Debian Codegraph @

include/deal.II/matrix_free/fe_evaluation.h

raw
1 // ------------------------------------------------------------------------
2 //
3 // SPDX-License-Identifier: LGPL-2.1-or-later
4 // Copyright (C) 2012 - 2025 by the deal.II authors
5 //
6 // This file is part of the deal.II library.
7 //
8 // Part of the source code is dual licensed under Apache-2.0 WITH
9 // LLVM-exception OR LGPL-2.1-or-later. Detailed license information
10 // governing the source code and code contributions can be found in
11 // LICENSE.md and CONTRIBUTING.md at the top level directory of deal.II.
12 //
13 // ------------------------------------------------------------------------
14
15
16 #ifndef dealii_matrix_free_fe_evaluation_h
17 #define dealii_matrix_free_fe_evaluation_h
18
19
20 #include <deal.II/base/config.h>
21
22 #include <deal.II/base/array_view.h>
23 #include <deal.II/base/exceptions.h>
24 #include <deal.II/base/observer_pointer.h>
25 #include <deal.II/base/symmetric_tensor.h>
26 #include <deal.II/base/template_constraints.h>
27 #include <deal.II/base/vectorization.h>
28
29 #include <deal.II/lac/vector_operation.h>
30
31 #include <deal.II/matrix_free/evaluation_flags.h>
32 #include <deal.II/matrix_free/evaluation_kernels.h>
33 #include <deal.II/matrix_free/evaluation_kernels_face.h>
34 #include <deal.II/matrix_free/evaluation_template_factory.h>
35 #include <deal.II/matrix_free/fe_evaluation_data.h>
36 #include <deal.II/matrix_free/hanging_nodes_internal.h>
37 #include <deal.II/matrix_free/mapping_data_on_the_fly.h>
38 #include <deal.II/matrix_free/matrix_free.h>
39 #include <deal.II/matrix_free/shape_info.h>
40 #include <deal.II/matrix_free/tensor_product_kernels.h>
41 #include <deal.II/matrix_free/type_traits.h>
42 #include <deal.II/matrix_free/vector_access_internal.h>
43
44 #include <type_traits>
45
46
47 DEAL_II_NAMESPACE_OPEN
48
49
50
51 /**
52 * This is the base class for the FEEvaluation classes. This class needs
53 * usually not be called in user code and does not have any public
54 * constructor. The usage is through the class FEEvaluation instead. It
55 * implements a reinit method that is used to set pointers so that operations
56 * on quadrature points can be performed quickly, access functions to vectors
57 * for the FEEvaluationBase::read_dof_values(),
58 * FEEvaluationBase::set_dof_values(), and
59 * FEEvaluationBase::distribute_local_to_global() functions, as well as
60 * methods to access values and gradients of finite element functions. It also
61 * inherits the geometry access functions provided by the class
62 * FEEvaluationData.
63 *
64 * This class has five template arguments:
65 *
66 * @tparam dim Dimension in which this class is to be used
67 *
68 * @tparam n_components Number of vector components when solving a system of
69 * PDEs. If the same operation is applied to several components of a PDE (e.g.
70 * a vector Laplace equation), they can be applied simultaneously with one
71 * call (and often more efficiently)
72 *
73 * @tparam Number Number format, usually @p double or @p float
74 *
75 * @tparam is_face Whether the class is used for a cell integrator (with
76 * quadrature dimension the same as the space dimension) or for a face
77 * integrator (with quadrature dimension one less)
78 *
79 * @tparam VectorizedArrayType Type of array to be worked on in a vectorized
80 * fashion, defaults to VectorizedArray<Number>
81 *
82 * @note Currently only VectorizedArray<Number, width> is supported as
83 * VectorizedArrayType.
84 *
85 *
86 * @ingroup matrixfree
87 */
88 template <int dim,
89 int n_components_,
90 typename Number,
91 bool is_face,
92 typename VectorizedArrayType>
93 class FEEvaluationBase
94 : public FEEvaluationData<dim, VectorizedArrayType, is_face>
95 {
96 public:
97 using number_type = Number;
98 using value_type =
99 std::conditional_t<n_components_ == 1,
100 VectorizedArrayType,
101 Tensor<1, n_components_, VectorizedArrayType>>;
102 using gradient_type = std::conditional_t<
103 n_components_ == 1,
104 Tensor<1, dim, VectorizedArrayType>,
105 std::conditional_t<
106 n_components_ == dim,
107 Tensor<2, dim, VectorizedArrayType>,
108 Tensor<1, n_components_, Tensor<1, dim, VectorizedArrayType>>>>;
109 using hessian_type = std::conditional_t<
110 n_components_ == 1,
111 Tensor<2, dim, VectorizedArrayType>,
112 std::conditional_t<
113 n_components_ == dim,
114 Tensor<3, dim, VectorizedArrayType>,
115 Tensor<1, n_components_, Tensor<2, dim, VectorizedArrayType>>>>;
116 static constexpr unsigned int dimension = dim;
117 static constexpr unsigned int n_components = n_components_;
118 static constexpr unsigned int n_lanes = VectorizedArrayType::size();
119
120 /**
121 * @name Reading from and writing to vectors
122 */
123 /** @{ */
124 /**
125 * For the vector @p src, read out the values on the degrees of freedom of
126 * the current cell, and store them internally. Similar functionality as the
127 * function DoFCellAccessor::get_interpolated_dof_values() when no constraints
128 * are present other than constraints from hanging nodes. Additionally, this
129 * function also resolves homogeneous constraints. Note that if vectorization
130 * is enabled, the DoF values for several cells are set.
131 *
132 * If some constraints on the vector are inhomogeneous, use the function
133 * FEEvaluationBase::read_dof_values_plain() instead and provide the vector
134 * with useful data also in constrained positions by calling
135 * AffineConstraints::distribute(). When accessing vector entries during the
136 * solution of linear systems, the temporary solution should always have
137 * homogeneous constraints and this method is the correct one. Further
138 * information on how to apply inhomogeneous constraints with matrix-free
139 * operators can be found in the respective section of step-37.
140 *
141 * If the given vector template class is a block vector (determined through
142 * the template function 'IsBlockVector<VectorType>::value', which checks
143 * for vectors derived from dealii::BlockVectorBase) or an
144 * std::vector<VectorType> or std::vector<VectorType *>, this function reads
145 * @p n_components blocks from the block vector starting at the index
146 * @p first_index. For non-block vectors, @p first_index is ignored.
147 *
148 * @note If this class was constructed without a MatrixFree object and the
149 * information is acquired on the fly through a
150 * DoFHandler<dim>::cell_iterator, only one single cell is used by this
151 * class and this function extracts the values of the underlying components
152 * on the given cell. This call is slower than the ones done through a
153 * MatrixFree object and lead to a structure that does not effectively use
154 * vectorization in the evaluate routines based on these values (instead,
155 * VectorizedArray::size() same copies are worked on).
156 */
157 template <typename VectorType>
158 void
159 read_dof_values(
160 const VectorType &src,
161 const unsigned int first_index = 0,
162 const std::bitset<n_lanes> &mask = std::bitset<n_lanes>().flip());
163
164 /**
165 * For the vector @p src, read out the values on the degrees of freedom of
166 * the current cell, and store them internally. Similar functionality as the
167 * function DoFCellAccessor::get_interpolated_dof_values(). As opposed to the
168 * FEEvaluationBase::read_dof_values() function, this function reads out the
169 * plain entries from vectors, without taking stored constraints into account.
170 * This way of access is appropriate when the constraints have been
171 * distributed on the vector by a call to AffineConstraints::distribute()
172 * previously. This function is also necessary when inhomogeneous constraints
173 * are to be used, as MatrixFree can only handle homogeneous constraints.
174 * Further information on how to apply inhomogeneous constraints with
175 * matrix-free operators can be found in the respective section of step-37.
176 * Note that if vectorization is enabled, the DoF values for several cells are
177 * set.
178 *
179 * If the given vector template class is a block vector (determined through
180 * the template function 'IsBlockVector<VectorType>::value', which checks
181 * for vectors derived from dealii::BlockVectorBase) or an
182 * std::vector<VectorType> or std::vector<VectorType *>, this function reads
183 * @p n_components blocks from the block vector starting at the index
184 * @p first_index. For non-block vectors, @p first_index is ignored.
185 *
186 * @note If this class was constructed without a MatrixFree object and the
187 * information is acquired on the fly through a
188 * DoFHandler<dim>::cell_iterator, only one single cell is used by this
189 * class and this function extracts the values of the underlying components
190 * on the given cell. This call is slower than the ones done through a
191 * MatrixFree object and lead to a structure that does not effectively use
192 * vectorization in the evaluate routines based on these values (instead,
193 * VectorizedArray::size() same copies are worked on).
194 */
195 template <typename VectorType>
196 void
197 read_dof_values_plain(
198 const VectorType &src,
199 const unsigned int first_index = 0,
200 const std::bitset<n_lanes> &mask = std::bitset<n_lanes>().flip());
201
202 /**
203 * Takes the values stored internally on dof values of the current cell and
204 * sums them into the vector @p dst. The function also applies constraints
205 * during the write operation. The functionality is hence similar to the
206 * function AffineConstraints::distribute_local_to_global. If vectorization
207 * is enabled, the DoF values for several cells are used.
208 *
209 * If the given vector template class is a block vector (determined through
210 * the template function 'IsBlockVector<VectorType>::value', which checks
211 * for vectors derived from dealii::BlockVectorBase) or an
212 * std::vector<VectorType> or std::vector<VectorType *>, this function
213 * writes to @p n_components blocks of the block vector starting at the
214 * index @p first_index. For non-block vectors, @p first_index is ignored.
215 *
216 * The @p mask can be used to suppress the write access for some of the
217 * cells contained in the current cell vectorization batch, e.g. in case of
218 * local time stepping, where some cells are excluded from a call. A value
219 * of `true` in the bitset means that the respective lane index will be
220 * processed, whereas a value of `false` skips this index. The default
221 * setting is a bitset that contains all ones, which will write the
222 * accumulated integrals to all cells in the batch.
223 *
224 * @note If this class was constructed without a MatrixFree object and the
225 * information is acquired on the fly through a
226 * DoFHandler<dim>::cell_iterator, only one single cell is used by this
227 * class and this function extracts the values of the underlying components
228 * on the given cell. This call is slower than the ones done through a
229 * MatrixFree object and lead to a structure that does not effectively use
230 * vectorization in the evaluate routines based on these values (instead,
231 * VectorizedArray::size() same copies are worked on).
232 */
233 template <typename VectorType>
234 void
235 distribute_local_to_global(
236 VectorType &dst,
237 const unsigned int first_index = 0,
238 const std::bitset<n_lanes> &mask = std::bitset<n_lanes>().flip()) const;
239
240 /**
241 * Takes the values stored internally on dof values of the current cell and
242 * writes them into the vector @p dst. The function skips the degrees of
243 * freedom which are constrained. As opposed to the
244 * distribute_local_to_global method, the old values at the position given
245 * by the current cell are overwritten. Thus, if a degree of freedom is
246 * associated to more than one cell (as usual in continuous finite
247 * elements), the values will be overwritten and only the value written last
248 * is retained. Please note that in a parallel context this function might
249 * also touch degrees of freedom owned by other MPI processes, so that a
250 * subsequent update or accumulation of ghost values as done by
251 * MatrixFree::loop() might invalidate the degrees of freedom set by this
252 * function.
253 *
254 * If the given vector template class is a block vector (determined through
255 * the template function 'IsBlockVector<VectorType>::value', which checks
256 * for vectors derived from dealii::BlockVectorBase) or an
257 * std::vector<VectorType> or std::vector<VectorType *>, this function
258 * writes to @p n_components blocks of the block vector starting at the
259 * index @p first_index. For non-block vectors, @p first_index is ignored.
260 *
261 * The @p mask can be used to suppress the write access for some
262 * of the cells contained in the current cell vectorization batch, e.g. in
263 * case of local time stepping, where some cells are excluded from a call.
264 * A value of `true` in the bitset means that the respective lane index will
265 * be processed, whereas a value of `false` skips this index. The default
266 * setting is a bitset that contains all ones, which will write the
267 * accumulated integrals to all cells in the batch.
268 *
269 * @note If this class was constructed without a MatrixFree object and the
270 * information is acquired on the fly through a
271 * DoFHandler<dim>::cell_iterator, only one single cell is used by this
272 * class and this function extracts the values of the underlying components
273 * on the given cell. This call is slower than the ones done through a
274 * MatrixFree object and lead to a structure that does not effectively use
275 * vectorization in the evaluate routines based on these values (instead,
276 * VectorizedArray::size() same copies are worked on).
277 */
278 template <typename VectorType>
279 void
280 set_dof_values(
281 VectorType &dst,
282 const unsigned int first_index = 0,
283 const std::bitset<n_lanes> &mask = std::bitset<n_lanes>().flip()) const;
284
285 /**
286 * Same as set_dof_values(), but without resolving constraints.
287 */
288 template <typename VectorType>
289 void
290 set_dof_values_plain(
291 VectorType &dst,
292 const unsigned int first_index = 0,
293 const std::bitset<n_lanes> &mask = std::bitset<n_lanes>().flip()) const;
294
295 /** @} */
296
297 /**
298 * @name Access to data at quadrature points or the data at cell DoFs
299 */
300 /** @{ */
301 /**
302 * Return the value stored in the array of coefficients for the elemental
303 * finite element solution expansion for the local degree of freedom with
304 * index @p dof. If the object is vector-valued, a vector-valued return
305 * argument is given. Thus, the argument @p dof can at most run until @p
306 * dofs_per_component rather than @p dofs_per_cell since the different
307 * components of a vector-valued FE are returned together. Note that when
308 * vectorization is enabled, values from several cells are grouped together
309 * in the inner VectorizedArray argument of @p value_type. If @p
310 * submit_dof_value was called last, the value corresponds to the data set
311 * there for the respective index. If @p integrate was called last, it
312 * instead corresponds to the value of the integrated function with the test
313 * function of the given index that gets written into the global vector by
314 * distribute_local_to_global().
315 */
316 value_type
317 get_dof_value(const unsigned int dof) const;
318
319 /**
320 * Write a value to the field containing coefficients associated with the
321 * cell-local degrees of freedom with index @p dof. This function writes to
322 * the same field as is accessed through @p get_dof_value. Therefore, the
323 * original data that is stored in this location, e.g. from reading a global
324 * vector via read_dof_values(), is overwritten as soon as a value is
325 * submitted for a particular DoF index.
326 */
327 void
328 submit_dof_value(const value_type val_in, const unsigned int dof);
329
330 /**
331 * Return the value of a finite element function interpolated to the
332 * quadrature point with index @p q_point after a call to
333 * @p evaluate() with @p EvaluationFlags::values set, or the
334 * value that has been stored there with a call to
335 * FEEvaluationBase::submit_value(). If the object is vector-valued, a
336 * vector-valued return argument is given. In case vectorization is enabled,
337 * values from several cells are grouped together as a VectorizedArray for
338 * each component in @p value_type.
339 */
340 value_type
341 get_value(const unsigned int q_point) const;
342
343 /**
344 * Write a contribution that gets multiplied by the value of the test
345 * function to the field containing the values at quadrature points with
346 * index @p q_point. As part of this function, the data gets multiplied by
347 * the quadrature weight and possible Jacobian determinants. When this
348 * function has been called for all quadrature points and a subsequent call
349 * to the function @p integrate(EvaluationFlags::values) has been made, the
350 * result is an array of entries, each representing the result of the
351 * integral of product of the test function multiplied by the data passed to
352 * this function.
353 *
354 * @note This function accesses the same field as through get_value(), so
355 * make sure to not call it after calling submit_value() for a specific
356 * quadrature point index.
357 */
358 void
359 submit_value(const value_type val_in, const unsigned int q_point);
360
361 /**
362 * For scalar elements, the value_type and gradient_type can be
363 * unintentionally mixed up because FEEvaluationBase does not distinguish
364 * between scalar accessors and vector-valued accessors and the respective
365 * types, but solely in terms of the number of components and dimension. Thus,
366 * enable the use of submit_value() also for tensors with a single component.
367 */
368 template <int n_components_local = n_components,
369 typename = std::enable_if_t<n_components == n_components_local>>
370 void
371 submit_value(const Tensor<1, 1, VectorizedArrayType> val_in,
372 const unsigned int q_point);
373
374 /**
375 * Return the gradient of the finite element function evaluated at
376 * quadrature point with index @p q_point after a call to
377 * @p evaluate() with @p EvaluationFlags::gradients set, collecting
378 * all components in a vector-valued problem as the outer tensor index and
379 * all partial derivatives as the inner (second) tensor index. For scalar
380 * problems, @p gradient_type is overloaded as a Tensor@<1, dim@>. For
381 * further information, see the get_value() function.
382 */
383 gradient_type
384 get_gradient(const unsigned int q_point) const;
385
386 /**
387 * Return the derivative of a finite element function interpolated to the
388 * quadrature point with index @p q_point after a call to
389 * @p evaluate(EvaluationFlags::gradients) the direction normal
390 * to the face: $\boldsymbol \nabla u(\mathbf x_q) \cdot \mathbf n(\mathbf
391 * x_q)$.
392 *
393 * This call is equivalent to calling get_gradient() * normal_vector()
394 * but will use a more efficient internal representation of data.
395 */
396 value_type
397 get_normal_derivative(const unsigned int q_point) const;
398
399 /**
400 * Write a contribution that gets multiplied by the gradient of the test
401 * function to the field containing the gradients at quadrature points with
402 * index @p q_point. When this function has queued information for all
403 * quadrature points and followed by a call to the function
404 * @p integrate(EvaluationFlags::gradients), the result is an
405 * array of entries, each representing the result of the integral of product
406 * of the test function gradient multiplied by the values passed to this
407 * function.
408 *
409 * @note This function accesses the same field as through get_gradient(),
410 * so make sure to not call it after calling submit_gradient() for a
411 * specific quadrature point index.
412 */
413 void
414 submit_gradient(const gradient_type grad_in, const unsigned int q_point);
415
416 /**
417 * In 1D, the value_type and gradient_type can be unintentionally mixed
418 * up because FEEvaluationBase does not distinguish between scalar accessors
419 * and vector-valued accessors and the respective types, but solely in terms
420 * of the number of components and dimension. Thus, enable the use of
421 * submit_gradient() also for rank-2 tensors with a single component.
422 */
423 template <int dim_ = dim,
424 typename = std::enable_if_t<dim_ == 1 && n_components == dim_>>
425 void
426 submit_gradient(const Tensor<2, 1, VectorizedArrayType> val_in,
427 const unsigned int q_point);
428
429 /**
430 * Write a contribution that gets multiplied by the gradient of the test
431 * function times the normal vector to the field containing the gradients at
432 * quadrature points with index @p q_point, see submit_gradient() for
433 * further information.
434 *
435 * @note This operation writes the data to the same field as
436 * submit_gradient(), submit_divergence(), submit_symmetric_gradient() and
437 * submit_curl(). As a consequence, only one of these functions can be
438 * used. Usually, the contribution of a potential call to this function must
439 * be added into the contribution for submit_gradient() when both are
440 * desired. Furthermore, the data array is the same as for get_gradient()
441 * and get_normal_derivative(), so invalid data will be returned after
442 * calling this function.
443 */
444 void
445 submit_normal_derivative(const value_type grad_in,
446 const unsigned int q_point);
447
448 /**
449 * Return the Hessian of the finite element function interpolated to
450 * quadrature point with index @p q_point after a call to
451 * @p evaluate(EvaluationFlags::hessians). If only the diagonal
452 * or even the trace of the Hessian, the Laplacian, is needed, use the other
453 * functions below.
454 */
455 hessian_type
456 get_hessian(const unsigned int q_point) const;
457
458 /**
459 * Return the diagonal of the Hessian of the finite element function
460 * interpolated to the quadrature point with index @p q_point after a call
461 * to @p evaluate(EvaluationFlags::hessians).
462 */
463 gradient_type
464 get_hessian_diagonal(const unsigned int q_point) const;
465
466 /**
467 * Return the Laplacian (i.e., the trace of the Hessian) of the finite
468 * element function interpolated to the quadrature point with index
469 * @p q_point after a call to @p evaluate(EvaluationFlags::hessians).
470 * Compared to the case when computing the full Hessian, some operations can
471 * be saved when only the Laplacian is requested.
472 */
473 value_type
474 get_laplacian(const unsigned int q_point) const;
475
476 /**
477 * Return the second derivative along the normal direction $\partial_{n}^2
478 * u_h$ (i.e., the Hessian of the function $u_h$ contracted twice with the
479 * direction of the normal vector) of the finite element
480 * function interpolated to the quadrature point with index
481 * @p q_point after a call to @p evaluate(EvaluationFlags::hessians).
482 * Compared to the case when computing the full Hessian, some operations can
483 * be saved when only the normal Hessian is requested.
484 */
485 value_type
486 get_normal_hessian(const unsigned int q_point) const;
487
488 /**
489 * Write a contribution that gets multiplied by the Hessian of the test
490 * function to the field containing the Hessians at quadrature points with
491 * index @p q_point. When this function has queued information for all
492 * quadrature points and followed by a call to the function
493 * @p integrate(EvaluationFlags::hessians), the result is an
494 * array of entries, each representing the result of the integral of product
495 * of the test function Hessian multiplied by the values passed to this
496 * function.
497 *
498 * @note This function accesses the same field as through get_hessian(),
499 * so make sure to not call it after calling submit_hessian() for a
500 * specific quadrature point index.
501 */
502 void
503 submit_hessian(const hessian_type hessian_in, const unsigned int q_point);
504
505 /**
506 * Write a contribution that gets multiplied by the Hessian of the test
507 * function times the normal projector to the field containing the Hessians at
508 * quadrature points with index @p q_point. When this function has queued
509 * information for all quadrature points and followed by a call to the
510 * function @p integrate(EvaluationFlags::hessians), the result is an
511 * array of entries, each representing the result of the integral of product
512 * of the test function Hessian multiplied by the values times the normal
513 * projector passed to this function.
514 *
515 * @note This function accesses the same field as through get_hessian() and
516 * get_normal_hessian() so make sure to not call it after calling
517 * submit_hessian() or submit_normal_hessian() for a specific quadrature point
518 * index.
519 */
520 void
521 submit_normal_hessian(const value_type normal_hessian_in,
522 const unsigned int q_point);
523
524 /**
525 * Return the divergence of a vector-valued finite element at quadrature
526 * point number @p q_point after a call to
527 * @p evaluate(EvaluationFlags::gradients).
528 *
529 * @note Only available for the vector-valued case (`n_components == dim`).
530 */
531 template <int dim_ = dim, typename = std::enable_if_t<n_components_ == dim_>>
532 VectorizedArrayType
533 get_divergence(const unsigned int q_point) const;
534
535 /**
536 * Write a contribution that is multiplied by the divergence of the test
537 * function to the field containing the gradients at quadrature points with
538 * index @p q_point. See submit_gradient() for further information.
539 *
540 * @note Only available for the vector-valued case (n_components == dim).
541 *
542 * @note This operation writes the data to the same field as
543 * submit_gradient(), submit_normal_derivative(),
544 * submit_symmetric_gradient() and submit_curl(). As a consequence, only one
545 * of these functions can be used. In case several terms of this kind appear
546 * in a weak form, the contribution of a potential call to this function
547 * must be added into the diagonal of the rank-2 tensor contribution passed
548 * to submit_gradient().
549 */
550 template <int dim_ = dim, typename = std::enable_if_t<n_components_ == dim_>>
551 void
552 submit_divergence(const VectorizedArrayType div_in,
553 const unsigned int q_point);
554
555 /**
556 * Return the symmetric gradient of the vector-valued finite element
557 * function interpolated at the quadrature point with index @p q_point after
558 * a call to @p evaluate(EvaluationFlags::gradients). It corresponds to
559 * <tt>0.5 (grad+grad<sup>T</sup>)</tt>.
560 *
561 * @note Only available for the vector-valued case (n_components == dim).
562 */
563 template <int dim_ = dim, typename = std::enable_if_t<n_components_ == dim_>>
564 SymmetricTensor<2, dim, VectorizedArrayType>
565 get_symmetric_gradient(const unsigned int q_point) const;
566
567 /**
568 * Write a contribution that is multiplied by the symmetric gradient of the
569 * test function to the field containing the gradients at quadrature points
570 * with index @p q_point. See submit_gradient() for further information.
571 *
572 * @note Only available for the vector-valued case (n_components == dim).
573 *
574 * @note This operation writes the data to the same field as
575 * submit_gradient(), submit_normal_derivative(), submit_divergence() and
576 * submit_curl(). As a consequence, only one of these functions can be
577 * used. In case several terms of this kind appear in a weak form, the
578 * contribution of a potential call to this function must be added into the
579 * diagonal of the rank-2 tensor contribution passed to submit_gradient(),
580 * in order not to overwrite information.
581 */
582 template <int dim_ = dim, typename = std::enable_if_t<n_components_ == dim_>>
583 void
584 submit_symmetric_gradient(
585 const SymmetricTensor<2, dim, VectorizedArrayType> grad_in,
586 const unsigned int q_point);
587
588 /**
589 * Return the curl of the vector field, $\nabla \times v$ interpolated to
590 * the quadrature point index after calling
591 * @p evaluate(EvaluationFlags::gradients).
592 *
593 * @note Only available for the vector-valued case (n_components == dim) in
594 * 2 and 3 dimensions.
595 */
596 template <int dim_ = dim,
597 typename = std::enable_if_t<n_components_ == dim_ && dim_ != 1>>
598 Tensor<1, (dim == 2 ? 1 : dim), VectorizedArrayType>
599 get_curl(const unsigned int q_point) const;
600
601 /**
602 * Write the components of a curl containing the values on quadrature point
603 * @p q_point. Access to the same data field as through get_gradient().
604 *
605 * @note Only available for the vector-valued case (n_components == dim).
606 *
607 * @note This operation writes the data to the same field as
608 * submit_gradient(), submit_normal_derivative(), submit_divergence() and
609 * submit_symmetric_gradient(). As a consequence, only one of these
610 * functions can be used. Usually, the contribution of a potential call to
611 * this function must be added to the respective entries of the rank-2
612 * tensor for submit_gradient().
613 */
614 template <int dim_ = dim,
615 typename = std::enable_if_t<n_components_ == dim_ && dim != 1>>
616 void
617 submit_curl(const Tensor<1, dim == 2 ? 1 : dim, VectorizedArrayType> curl_in,
618 const unsigned int q_point);
619
620 /**
621 * Take values collected at quadrature points via the submit_value() function,
622 * multiply by the Jacobian determinant
623 * and quadrature weights (JxW) and sum the values for all quadrature
624 * points on the cell. The result is a scalar, representing the integral
625 * over the function over the cell. If a vector-element is used, the
626 * resulting components are still separated. Moreover, if vectorization is
627 * enabled, the integral values of several cells are contained in the slots
628 * of the returned VectorizedArray field.
629 *
630 * @note In case the FEEvaluation object is initialized with a batch of
631 * cells where not all lanes in the SIMD vector VectorizedArray are
632 * representing actual data, this method performs computations on dummy data
633 * (that is copied from the last valid lane) that will not make sense. Thus,
634 * the user needs to make sure that it is not used in any computation
635 * explicitly, like when summing the results of several cells.
636 */
637 value_type
638 integrate_value() const;
639
640 /** @} */
641
642 /**
643 * Return the underlying MatrixFree object.
644 */
645 const MatrixFree<dim, Number, VectorizedArrayType> &
646 get_matrix_free() const;
647
648 protected:
649 /**
650 * Constructor. Made protected to prevent users from directly using this
651 * class. Takes all data stored in MatrixFree. If applied to problems with
652 * more than one finite element or more than one quadrature formula selected
653 * during construction of @p matrix_free, @p dof_no, @p
654 * first_selected_component and @p quad_no allow to select the appropriate
655 * components.
656 */
657 FEEvaluationBase(
658 const MatrixFree<dim, Number, VectorizedArrayType> &matrix_free,
659 const unsigned int dof_no,
660 const unsigned int first_selected_component,
661 const unsigned int quad_no,
662 const unsigned int fe_degree,
663 const unsigned int n_q_points,
664 const bool is_interior_face,
665 const unsigned int active_fe_index,
666 const unsigned int active_quad_index,
667 const unsigned int face_type);
668
669 /**
670 * Constructor that comes with reduced functionality and works similar as
671 * FEValues. The arguments are similar to the ones passed to the constructor
672 * of FEValues, with the notable difference that FEEvaluation expects a
673 * one-dimensional
674 * quadrature formula, Quadrature<1>, instead of a @p dim
675 * dimensional one. The finite element can be both scalar or vector valued,
676 * but this method always only selects a scalar base element at a time (with
677 * @p n_components copies as specified by the class template argument). For
678 * vector-valued elements, the optional argument @p first_selected_component
679 * allows to specify the index of the base element to be used for
680 * evaluation. Note that the internal data structures always assume that the
681 * base element is primitive, non-primitive are not supported currently.
682 *
683 * As known from FEValues, a call to the reinit method with a
684 * Triangulation::cell_iterator is necessary to make the geometry and
685 * degrees of freedom of the current class known. If the iterator includes
686 * DoFHandler information (i.e., it is a DoFHandler::cell_iterator or
687 * similar), the initialization allows to also read from or write to vectors
688 * in the standard way for DoFHandler::active_cell_iterator types for one
689 * cell at a time. However, this approach is much slower than the path with
690 * MatrixFree with MPI since index translation has to be done. As only one
691 * cell at a time is used, this method does not vectorize over several
692 * elements (which is most efficient for vector operations), but only
693 * possibly within the element if the evaluate/integrate routines are
694 * combined inside user code (e.g. for computing cell matrices).
695 *
696 * The optional FEEvaluationData object allows several
697 * FEEvaluation objects to share the geometry evaluation, i.e., the
698 * underlying mapping and quadrature points do only need to be evaluated
699 * once. This only works if the quadrature formulas are the same. Otherwise,
700 * a new evaluation object is created. Make sure to not pass an optional
701 * object around when you intend to use the FEEvaluation object in %parallel
702 * with another one because otherwise the intended sharing may create race
703 * conditions.
704 */
705 FEEvaluationBase(
706 const Mapping<dim> &mapping,
707 const FiniteElement<dim> &fe,
708 const Quadrature<1> &quadrature,
709 const UpdateFlags update_flags,
710 const unsigned int first_selected_component,
711 const FEEvaluationData<dim, VectorizedArrayType, is_face> *other);
712
713 /**
714 * Copy constructor. If FEEvaluationBase was constructed from a mapping, fe,
715 * quadrature, and update flags, the underlying geometry evaluation based on
716 * FEValues will be deep-copied in order to allow for using in parallel with
717 * threads.
718 */
719 FEEvaluationBase(const FEEvaluationBase &other);
720
721 /**
722 * Copy assignment operator. If FEEvaluationBase was constructed from a
723 * mapping, fe, quadrature, and update flags, the underlying geometry
724 * evaluation based on FEValues will be deep-copied in order to allow for
725 * using in parallel with threads.
726 */
727 FEEvaluationBase &
728 operator=(const FEEvaluationBase &other);
729
730 /**
731 * Destructor.
732 */
733 ~FEEvaluationBase();
734
735 /**
736 * A unified function to read from and write into vectors based on the given
737 * template operation. It can perform the operation for @p read_dof_values,
738 * @p distribute_local_to_global, and @p set_dof_values. It performs the
739 * operation for several vectors at a time.
740 */
741 template <typename VectorType, typename VectorOperation>
742 void
743 read_write_operation(
744 const VectorOperation &operation,
745 const std::array<VectorType *, n_components_> &vectors,
746 const std::array<
747 const std::vector<ArrayView<const typename VectorType::value_type>> *,
748 n_components_> &vectors_sm,
749 const std::bitset<n_lanes> &mask,
750 const bool apply_constraints = true) const;
751
752 /**
753 * A unified function to read from and write into vectors based on the given
754 * template operation for DG-type schemes where all degrees of freedom on
755 * cells are contiguous. It can perform the operation for read_dof_values(),
756 * distribute_local_to_global(), and set_dof_values() for several vectors at
757 * a time, depending on n_components.
758 */
759 template <typename VectorType, typename VectorOperation>
760 void
761 read_write_operation_contiguous(
762 const VectorOperation &operation,
763 const std::array<VectorType *, n_components_> &vectors,
764 const std::array<
765 const std::vector<ArrayView<const typename VectorType::value_type>> *,
766 n_components_> &vectors_sm,
767 const std::bitset<n_lanes> &mask) const;
768
769 /**
770 * A unified function to read from and write into vectors based on the given
771 * template operation for the case when we do not have an underlying
772 * MatrixFree object. It can perform the operation for @p read_dof_values,
773 * @p distribute_local_to_global, and @p set_dof_values. It performs the
774 * operation for several vectors at a time, depending on n_components.
775 */
776 template <typename VectorType, typename VectorOperation>
777 void
778 read_write_operation_global(
779 const VectorOperation &operation,
780 const std::array<VectorType *, n_components_> &vectors) const;
781
782 /**
783 * Apply hanging-node constraints.
784 */
785 void
786 apply_hanging_node_constraints(const bool transpose) const;
787
788 /**
789 * This is the general array for all data fields.
790 */
791 AlignedVector<VectorizedArrayType> *scratch_data_array;
792
793 /**
794 * A pointer to the underlying data.
795 */
796 const MatrixFree<dim, Number, VectorizedArrayType> *matrix_free;
797
798 /**
799 * A temporary data structure necessary to read degrees of freedom when no
800 * MatrixFree object was given at initialization.
801 */
802 mutable std::vector<types::global_dof_index> local_dof_indices;
803 };
804
805
806
807 // backward compatibility
808 template <int dim,
809 int n_components_,
810 typename Number,
811 bool is_face,
812 typename VectorizedArrayType = VectorizedArray<Number>>
813 using FEEvaluationAccess DEAL_II_DEPRECATED =
814 FEEvaluationBase<dim, n_components_, Number, is_face, VectorizedArrayType>;
815
816 /**
817 * The class that provides all functions necessary to evaluate functions at
818 * quadrature points and cell integrations. In functionality, this class is
819 * similar to FEValues, however, it includes a lot of specialized functions
820 * that make it much faster (between 5 and 500, depending on the polynomial
821 * degree). For evaluation of face terms in DG, see the class
822 * FEFaceEvaluation.
823 *
824 * <h3>Usage and initialization</h3>
825 *
826 * <h4>Fast usage in combination with MatrixFree</h4>
827 *
828 * The first and foremost way of usage is to initialize this class from a
829 * MatrixFree object that caches everything related to the degrees of freedom
830 * and the mapping information. This way, it is possible to use vectorization
831 * for applying a differential operator for several cells at once.
832 *
833 * The capabilities of FEEvaluation span a large spectrum of integration tasks
834 * for weak forms. In general, there are two classes of tasks that get
835 * done. One is the @p evaluate path that interpolates from a solution vector
836 * to quadrature points:
837 *
838 * @code
839 * FEEvaluation<dim,fe_degree> fe_eval(matrix_free);
840 * for (unsigned int cell_index = cell_range.first;
841 * cell_index < cell_range.second; ++cell_index)
842 * {
843 * fe_eval.reinit(cell_index);
844 * fe_eval.read_dof_values(vector);
845 * fe_eval.evaluate(EvaluationFlags::values); // interpolate values only
846 * for (unsigned int q=0; q<fe_eval.n_q_points; ++q)
847 * {
848 * VectorizedArray<double> val = fe_eval.get_value(q);
849 * // do something with val
850 * }
851 * }
852 * @endcode
853 *
854 * Likewise, a gradient of the finite element solution represented by
855 * `vector` can be interpolated to the quadrature points by
856 * `fe_eval.get_gradient(q)`. The combination of read_dof_values(), evaluate()
857 * and get_value() is similar to what FEValues::get_function_values or
858 * FEValues::get_function_gradients does, but it is in general much faster
859 * because it makes use of the tensor product, see the description of the
860 * evaluation routines below, and can do this operation for several cells at
861 * once through vectorization.
862 *
863 * The second class of tasks done by FEEvaluation are integration tasks for
864 * right hand sides. In finite element computations, these typically consist
865 * of multiplying a quantity on quadrature points (a function value, or a
866 * field interpolated by the finite element space itself) by a set of test
867 * functions and integrating over the cell through summation of the values in
868 * each quadrature point, multiplied by the quadrature weight and the Jacobian
869 * determinant of the transformation. If a generic Function object is given
870 * and we want to compute $v_i = \int_\Omega \varphi_i f dx$, this is done by
871 * the following cell-wise integration:
872 *
873 * @code
874 * FEEvaluation<dim,fe_degree> fe_eval(matrix_free);
875 * Function<dim> &function = ...;
876 * for (unsigned int cell_index = cell_range.first;
877 * cell_index < cell_range.second; ++cell_index)
878 * {
879 * fe_eval.reinit(cell_index);
880 * for (unsigned int q=0; q<fe_eval.n_q_points; ++q)
881 * {
882 * const Point<dim,VectorizedArray<double> > p_vect =
883 * fe_eval.quadrature_point(q);
884 * // Need to evaluate function for each component in VectorizedArray
885 * VectorizedArray<double> f_value = 0.0;
886 * for (unsigned int v=0; v<VectorizedArray<double>::size(); ++v)
887 * {
888 * Point<dim> p;
889 * for (unsigned int d=0; d<dim; ++d)
890 * p[d] = p_vect[d][v];
891 * f_value[v] = function.value(p);
892 * }
893 * fe_eval.submit_value(f_value, q);
894 * }
895 * fe_eval.integrate(EvaluationFlags::values);
896 * fe_eval.distribute_local_to_global(dst);
897 * }
898 * @endcode
899 *
900 * In this code, the call to @p fe_eval.submit_value() prepares for the
901 * multiplication by the test function prior to the actual integration (inside
902 * the submit call, the value to be tested is also multiplied by the
903 * determinant of the Jacobian and the quadrature weight). In the
904 * @p integrate() call, an integral contribution tested by each basis function
905 * underlying the FEEvaluation object (e.g. the four linear shape functions of
906 * FE_Q@<2@>(1) in 2d) is computed, which gives the vector entries to be
907 * summed into the @p dst vector. Note that the above code needs to explicitly
908 * loop over the components in the vectorized array for evaluating the
909 * function, which is necessary for interfacing with a generic Function object
910 * with double arguments. Simple functions can also be implemented in
911 * VectorizedArray form directly as VectorizedArray provides the basic math
912 * operations.
913 *
914 * For evaluating a bilinear form, the evaluation on a source vector is
915 * combined with the integration involving test functions that get written
916 * into a result vector. This setting is the context of matrix-free operator
917 * evaluation and explained in the step-37 and step-48 tutorial programs.
918 *
919 * Note that the two vector accesses through FEEvaluation::read_dof_values and
920 * FEEvaluation::distribute_local_to_global resolve constraints on the fly,
921 * based on the AffineConstraints object specified at the MatrixFree::reinit()
922 * call. In case the values in the degrees of freedom are of interest (usually
923 * only the values in quadrature points are necessary), these can be accessed
924 * through FEEvaluation::get_dof_value(i), where i is the index of the basis
925 * function. Note that the numbering of the degrees of freedom for continuous
926 * elements in FEEvaluation is different from the ordering in FE_Q (or
927 * FEValues) because FEEvaluation needs to access them in lexicographic order,
928 * which is the ordering used in FE_DGQ, for instance. Re-indexing would be
929 * too expensive because the access inside evaluate() and integrate() is on
930 * the critical path in the tensorial evaluation parts. An alternative to
931 * filling the DoF values by read_dof_values() before an evaluate() call is to
932 * manually assign a value by a set_dof_value() call. Likewise, if the local
933 * result of integration should be further processed rather than scattered
934 * into a vector by distribute_local_to_global(), one can access it by
935 * get_dof_value() after an integrate() call. An example for using the values
936 * of an integral in a different context is fast assembly of matrices as shown
937 * in the next subsection.
938 *
939 * For most operator evaluation tasks that repeatedly go through the mesh, the
940 * realization by MatrixFree that combines pre-computed data for the mapping
941 * (Jacobian transformations for the geometry description) with on-the-fly
942 * evaluation of basis functions is the most efficient way of doing things. In
943 * other words, the framework selects a trade-off between memory usage and
944 * initialization of objects that is suitable for replacement of matrix-vector
945 * products or explicit time integration in a matrix-free way.
946 *
947 * <h4>Usage without pre-initialized MatrixFree object</h4>
948 *
949 * The second form of usage is to initialize FEEvaluation from geometry
950 * information generated by FEValues. This allows to apply the integration
951 * loops on the fly without prior initialization of MatrixFree objects. This
952 * can be useful when the memory and initialization cost of MatrixFree is not
953 * acceptable, e.g. when a different number of quadrature points should be
954 * used for one single evaluation in error computation. Also, when using the
955 * routines of this class to assemble matrices the trade-off implied by the
956 * MatrixFree class may not be desired. In such a case, the cost to initialize
957 * the necessary geometry data on the fly is comparably low and thus avoiding
958 * a global object MatrixFree can be useful. When used in this way, reinit
959 * methods reminiscent from FEValues with a cell iterator are used. However,
960 * note that this model results in working on a single cell at a time, with
961 * geometry data duplicated in all components of the vectorized array. Thus,
962 * vectorization is only useful when it can apply the same operation on
963 * different data, e.g. when performing matrix assembly.
964 *
965 * As an example, consider the following code to assemble the contributions to
966 * the Laplace matrix:
967 *
968 * @code
969 * FEEvaluation<dim,fe_degree> fe_eval (mapping, finite_element,
970 * QGauss<1>(fe_degree+1), flags);
971 * for (const auto &cell : dof_handler.active_cell_iterators())
972 * {
973 * fe_eval.reinit(cell);
974 * for (unsigned int i=0; i<dofs_per_cell;
975 * i += VectorizedArray<double>::size())
976 * {
977 * const unsigned int n_items =
978 * i+VectorizedArray<double>::size() > dofs_per_cell ?
979 * (dofs_per_cell - i) :
980 * VectorizedArray<double>::size();
981 *
982 * // Set n_items unit vectors
983 * for (unsigned int j=0; j<dofs_per_cell; ++j)
984 * fe_eval.set_dof_value(VectorizedArray<double>(), j);
985 * for (unsigned int v=0; v<n_items; ++v)
986 * {
987 * VectorizedArray<double> one_value = VectorizedArray<double>();
988 * one_value[v] = 1.;
989 * fe_eval.set_dof_value(one_value, i+v);
990 * }
991 *
992 * // Apply operator on unit vector to generate the next few matrix
993 * // columns
994 * fe_eval.evaluate(EvaluationFlags::values|EvaluationFlags::gradients);
995 * for (unsigned int q=0; q<n_q_points; ++q)
996 * {
997 * fe_eval.submit_value(10.*fe_eval.get_value(q), q);
998 * fe_eval.submit_gradient(fe_eval.get_gradient(q), q);
999 * }
1000 * fe_eval.integrate(EvaluationFlags::values|EvaluationFlags::gradients);
1001 *
1002 * // Insert computed entries in matrix
1003 * for (unsigned int v=0; v<n_items; ++v)
1004 * for (unsigned int j=0; j<dofs_per_cell; ++j)
1005 * cell_matrix(fe_eval.get_internal_dof_numbering()[j],
1006 * fe_eval.get_internal_dof_numbering()[i+v])
1007 * = fe_eval.get_dof_value(j)[v];
1008 * }
1009 * ...
1010 * }
1011 * @endcode
1012 *
1013 * This code generates the columns of the cell matrix with the loop over @p i
1014 * above. The way this is done is the following: FEEvaluation's routines focus
1015 * on the evaluation of finite element operators, so for computing a cell
1016 * matrix out of an operator evaluation it is applied to all the unit vectors
1017 * on the cell. Applying the operator on a unit vector might seem inefficient
1018 * but the evaluation routines used here are so quick that they still work
1019 * much faster than what is possible with FEValues. In particular, the
1020 * complexity is <code>(fe_degree+1)<sup>2*dim+1</sup> </code> rather than
1021 * <code>(fe_degree+1)<sup>3*dim</sup> </code>.
1022 *
1023 * Due to vectorization, we can generate matrix columns for several unit
1024 * vectors at a time (e.g. 4). The variable @p n_items make sure that we do
1025 * the last iteration where the number of cell dofs is not divisible by the
1026 * vectorization length correctly. Also note that we need to get the internal
1027 * dof numbering applied by fe_eval because FEEvaluation internally uses a
1028 * lexicographic numbering of degrees of freedom as explained above.
1029 *
1030 * <h4>Internal data organization</h4>
1031 *
1032 * The temporary data for holding the solution values on the local degrees of
1033 * freedom as well as the interpolated values, gradients, and Hessians on
1034 * quadrature points is a scratch array provided by
1035 * MatrixFree::acquire_scratch_data() that is re-used between different calls
1036 * to FEEvaluation. Therefore, constructing an FEEvaluation object is
1037 * typically cheap and does not involve any expensive operation. Only a few
1038 * dozen pointers to the actual data fields are set during
1039 * construction. Therefore, no negative performance impact arises when
1040 * creating an FEEvaluation several times per loop, such as at the top of a
1041 * `local_cell_operation` operation that is split in small chunks for a parallel
1042 * for loop, obviating a separate scratch data field for parallel loops as
1043 * necessary in the loop of @p WorkStream.
1044 *
1045 * When using the FEEvaluation class in multithreaded mode, the thread local
1046 * storage of the scratch data in MatrixFree automatically makes sure that
1047 * each thread gets it private data array. Note, however, that deal.II must be
1048 * compiled with thread support also when all the thread parallelization is
1049 * provided externally and not done via deal.II's routines, such as
1050 * OpenMP. This is because deal.II needs to know the notation of thread local
1051 * storage. The FEEvaluation kernels have been verified to work within OpenMP
1052 * loops.
1053 *
1054 * <h4>Vectorization scheme through VectorizedArray</h4>
1055 *
1056 * This class is designed to perform all arithmetics on single-instruction
1057 * multiple-data (SIMD) instructions present on modern CPUs by explicit
1058 * vectorization, which are made available in deal.II through the class
1059 * VectorizedArray, using the widest vector width available at
1060 * configure/compile time. In order to keep programs flexible, FEEvaluation
1061 * always applies vectorization over several elements. This is often the best
1062 * compromise because computations on different elements are usually
1063 * independent in the finite element method (except of course the process of
1064 * adding an integral contribution to a global residual vector), also in more
1065 * complicated scenarios: Stabilization parameter can e.g. be defined as the
1066 * maximum of some quantities on all quadrature points of a cell divided by
1067 * the cell's volume, but without locally mixing the results with
1068 * neighbors. Using the terminology from computer architecture, the design of
1069 * FEEvaluation relies on not doing any cross-lane data exchange when
1070 * operating on the cell in typical integration scenarios.
1071 *
1072 * When the number of cells in the problem is not a multiple of the number of
1073 * array elements in the SIMD vector, the implementation of FEEvaluation fills
1074 * in some dummy entries in the unused SIMD lanes and carries them around
1075 * nonetheless, a choice made necessary since the length of VectorizedArray is
1076 * fixed at compile time. Yet, this approach most often results in superior
1077 * code as compared to an auto-vectorization setup where an alternative
1078 * unvectorized code path would be necessary next to the vectorized version to
1079 * be used on fully populated lanes, together with a dispatch mechanism. In
1080 * @p read_dof_values, the empty lanes resulting from a reinit() call to an
1081 * incomplete batch of cells are set to zero, whereas
1082 * @p distribute_local_to_global or @p set_dof_values simply ignores the
1083 * content in the empty lanes. The number of actually filled SIMD lanes can by
1084 * queried by MatrixFree::n_active_entries_per_cell_batch() or
1085 * MatrixFree::n_active_entries_per_face_batch().
1086 *
1087 * Obviously, the computations performed on the artificial lanes (without real
1088 * data) should never be mixed with valid results. The contract in using this
1089 * class is that the user makes sure that lanes are not crossed in user code,
1090 * in particular since it is not clear a priori which cells are going to be
1091 * put together in vectorization. For example, results on an element should
1092 * not be added to results on other elements except through the global vector
1093 * access methods or by access that is masked by
1094 * MatrixFree::n_active_entries_per_cell_batch(). No guarantee can be made
1095 * that results on artificial lanes will always be zero that can safely be
1096 * added to other results: The data on JxW or Jacobians is copied from the
1097 * last valid lane in order to avoid division by zero that could trigger
1098 * floating point exceptions or trouble in other situations.
1099 *
1100 * <h3>Description of evaluation routines</h3>
1101 *
1102 * This class contains specialized evaluation routines for elements based on
1103 * tensor-product quadrature formulas and tensor-product-like shape functions,
1104 * including standard FE_Q or FE_DGQ elements and quadrature points symmetric
1105 * around 0.5 (like Gauss quadrature), FE_DGP elements based on truncated
1106 * tensor products as well as the faster case of Gauss-Lobatto elements with
1107 * Gauss-Lobatto quadrature which give diagonal mass matrices and quicker
1108 * evaluation internally. The main benefit of this class is the evaluation of
1109 * all shape functions in all quadrature or integration over all shape
1110 * functions in <code>dim (fe_degree+1)<sup>dim+1</sup> </code> operations
1111 * instead of the slower <code> (fe_degree+1)<sup>2*dim</sup></code>
1112 * complexity in the evaluation routines of FEValues. This is done by an
1113 * algorithm called sum factorization which factors out constant factors
1114 * during the evaluation along a coordinate direction. This algorithm is the
1115 * basis of many spectral element algorithms.
1116 *
1117 * Note that many of the operations available through this class are inherited
1118 * from the base class FEEvaluationBase, in particular reading from and
1119 * writing to vectors. Furthermore, functionality to access to values,
1120 * gradients and Hessians of the finite element function at quadrature points
1121 * is inherited.
1122 *
1123 * This class assumes that the shape functions of the FiniteElement under
1124 * consideration do <em>not</em> depend on the geometry of the cells in real
1125 * space. Currently, other finite elements cannot be treated with the
1126 * matrix-free concept.
1127 *
1128 * <h4>Degree of finite element as a compile-time parameter</h4>
1129 *
1130 * The class FEEvaluation as two usage models. The first usage model is to
1131 * specify the polynomial degree as a template parameter. This guarantees
1132 * maximum
1133 * efficiency: The evaluation with sum factorization performs a number of nested
1134 * short 1d loops of length equal to the polynomial degree plus one. If the
1135 * loop bounds are known at compile time, the compiler can unroll loops as
1136 * deemed most efficient by its heuristics. At least the innermost loop is
1137 * almost always completely unrolled, avoiding the loop overhead.
1138 *
1139 * However, carrying the polynomial degree (and the number of quadrature
1140 * points) as a template parameter makes things more complicated in codes
1141 * where different polynomial degrees should be considered, e.g. in
1142 * application codes where the polynomial degree is given through an input
1143 * file. The second usage model is to rely on pre-compiled code for polynomial
1144 * degrees. While a user code can use different functions for the cells (that
1145 * get e.g. invoked by some dynamic dispatch mechanism for the various degree
1146 * templates), deal.II also supports usage of this class based on the
1147 * information in the element passed to the initialization. For this usage
1148 * model, set the template parameter for the polynomial degree to -1 and
1149 * choose an arbitrary number for the number of quadrature points. That code
1150 * part contains pre-compiled templated code for polynomial degrees between 1
1151 * and 6 and common quadrature formulas, which runs almost as fast as the
1152 * templated version. In case the chosen degree is not pre-compiled, an
1153 * evaluator object with template specialization for -1 is invoked that runs
1154 * according to run-time bounds.
1155 *
1156 * An overview of the performance of FEEvaluation is given in the following
1157 * figure. It considers the time spent per degree of freedom for evaluating
1158 * the Laplacian with continuous finite elements using a code similar to the
1159 * step-37 tutorial program for single-precision arithmetics. The time is
1160 * based on an experiment on a single core of an Intel Xeon E5-2687W v4,
1161 * running at 3.4 GHz and measured at problem sizes of around 10 million. The
1162 * plot lists the computational time (around 0.1 seconds) divided by the
1163 * number of degrees freedom.
1164 *
1165 * @image html fe_evaluation_laplacian_time_per_dof.png
1166 *
1167 * The figure shows that the templated computational kernels are between 2.5
1168 * and 3 times faster than the non-templated ones. The fastest turnaround on
1169 * this setup is for polynomial degree 5 at 7.4e-9 seconds per degree of
1170 * freedom or 134 million degrees of freedom per second - on a single
1171 * core. The non-templated version is also fastest at polynomial degree 5 with
1172 * 2.1e-9 seconds per degree of freedom or 48 million degrees of freedom per
1173 * second. Note that using FEEvaluation with template `degree=-1` selects the
1174 * fast path for degrees between one and six, and the slow path for other
1175 * degrees.
1176 *
1177 * <h4>Pre-compiling code for more polynomial degrees</h4>
1178 *
1179 * It is also possible to pre-compile the code in FEEvaluation for a different
1180 * maximal polynomial degree. This is controlled by the class
1181 * internal::FEEvaluationFactory and the implementation in
1182 * `include/deal.II/matrix_free/evaluation_template_factory.templates.h`. By
1183 * setting the macro `FE_EVAL_FACTORY_DEGREE_MAX` to the desired integer and
1184 * instantiating the classes FEEvaluationFactory and FEFaceEvaluationFactory
1185 * (the latter for FEFaceEvaluation) creates paths to templated functions for
1186 * a possibly larger set of degrees. This can both be set when configuring
1187 * deal.II by passing the flag `-D FE_EVAL_FACTORY_DEGREE_MAX=8` (in case you
1188 * want to compile all degrees up to eight; recommended setting) or by
1189 * compiling `evaluation_template_factory.templates.h` and
1190 * `evaluation_template_face_factory.templates.h` with the
1191 * `FE_EVAL_FACTORY_DEGREE_MAX` overridden to the desired value. In the second
1192 * option, symbols will be available twice, and it depends on your linker and
1193 * dynamic library loader whether the user-specified setting takes precedence;
1194 * use `LD_PRELOAD` to select the desired library. You can check if fast
1195 * evaluation/integration for a given degree/n_quadrature_points pair by
1196 * calling FEEvaluation::fast_evaluation_supported() or
1197 * FEFaceEvaluation::fast_evaluation_supported().
1198 *
1199 * <h3>Handling multi-component systems</h3>
1200 *
1201 * FEEvaluation also allows for treating vector-valued problems through a
1202 * template parameter on the number of components:
1203 *
1204 * @code
1205 * FEEvaluation<dim,fe_degree,n_q_points_1d,n_components> fe_eval(matrix_free);
1206 * @endcode
1207 *
1208 * If used this way, the components can be gathered from several components of
1209 * an @p std::vector<VectorType> through the call
1210 *
1211 * @code
1212 * fe_eval.read_dof_values(src, 0);
1213 * @endcode
1214 *
1215 * where the 0 means that the vectors starting from the zeroth vector in the
1216 * @p std::vector should be used, <code>src[0], src[1], ...,
1217 * src[n_components-1]</code>.
1218 *
1219 * An alternative way for reading multi-component systems is possible if the
1220 * DoFHandler underlying the MatrixFree data is based on an FESystem of @p
1221 * n_components entries. In that case, a single vector is provided for the
1222 * read_dof_values() and distribute_local_to_global() calls.
1223 *
1224 * An important property of FEEvaluation in multi-component systems is the
1225 * layout of multiple components in the get_value(), get_gradient(), or
1226 * get_dof_value() calls. In this case, instead of a scalar return field
1227 * VectorizedArray@<double@> a tensor is returned,
1228 *
1229 * @code
1230 * get_value -> Tensor<1,n_components,VectorizedArray<double>>
1231 * get_gradient -> Tensor<1,n_components,Tensor<1,dim,VectorizedArray<double>>
1232 * @endcode
1233 *
1234 * In a similar vein, the submit_value() and submit_gradient() calls take
1235 * tensors of values. Note that there exist specializations of these types for
1236 * @p n_components=1 and @p n_components=dim. In the scalar case, these
1237 * provide the scalar return types described above. In the vector-valued case,
1238 * the gradient is converted from
1239 * <code>Tensor@<1,dim,Tensor@<1,dim,VectorizedArray@<double@> @> @></code> to
1240 * <code>Tensor@<2,dim,VectorizedArray@<double@> @></code>. Furthermore,
1241 * additional operations such as the divergence or curl are available.
1242 *
1243 * In case different shape functions are combined, for example mixed finite
1244 * element formulations in Stokes flow, two FEEvaluation objects are created,
1245 * one for the velocity and one for the pressure. Those are then combined on
1246 * quadrature points:
1247 *
1248 * @code
1249 * FEEvaluation<dim,degree_p+1,degree_p+2,dim> velocity (data, 0);
1250 * FEEvaluation<dim,degree_p, degree_p+2,1, > pressure (data, 1);
1251 *
1252 * for (unsigned int cell=cell_range.first; cell<cell_range.second; ++cell)
1253 * {
1254 * velocity.reinit (cell);
1255 * velocity.read_dof_values (src.block(0));
1256 * velocity.evaluate (EvaluationFlags::gradients);
1257 * pressure.reinit (cell);
1258 * pressure.read_dof_values (src.block(1));
1259 * pressure.evaluate (EvaluationFlags::values);
1260 *
1261 * for (unsigned int q=0; q<velocity.n_q_points; ++q)
1262 * {
1263 * SymmetricTensor<2,dim,VectorizedArray<double> > sym_grad_u =
1264 * velocity.get_symmetric_gradient (q);
1265 * VectorizedArray<double> pres = pressure.get_value(q);
1266 * VectorizedArray<double> div = -trace(sym_grad_u);
1267 * pressure.submit_value (div, q);
1268 *
1269 * // subtract p * I
1270 * for (unsigned int d=0; d<dim; ++d)
1271 * sym_grad_u[d][d] -= pres;
1272 *
1273 * velocity.submit_symmetric_gradient(sym_grad_u, q);
1274 * }
1275 *
1276 * velocity.integrate (EvaluationFlags::gradients);
1277 * velocity.distribute_local_to_global (dst.block(0));
1278 * pressure.integrate (EvaluationFlags::values);
1279 * pressure.distribute_local_to_global (dst.block(1));
1280 * }
1281 * @endcode
1282 *
1283 * This code assumes that a BlockVector of two components describes the
1284 * velocity and pressure components, respectively. For identifying the
1285 * different DoFHandler objects for velocity and pressure, the second argument
1286 * to the FEEvaluation objects specify the respective component 0 for velocity
1287 * and 1 for pressure. For further examples of vector-valued problems, the
1288 * deal.II test suite includes a few additional examples as well, e.g. the
1289 * Stokes operator described above is found at
1290 * https://github.com/dealii/dealii/blob/master/tests/matrix_free/matrix_vector_stokes_noflux.cc
1291 *
1292 * <h3>Handling several integration tasks and data storage in quadrature
1293 * points</h3>
1294 *
1295 * The design of FEEvaluation and MatrixFree separates the geometry from the
1296 * basis functions. Therefore, several DoFHandler objects (or the same
1297 * DoFHandler equipped with different constraint objects) can share the same
1298 * geometry information like in the Stokes example above. All geometry is
1299 * cached once in MatrixFree, so FEEvaluation does not need to do expensive
1300 * initialization calls and rather sets a few pointers. This realization is
1301 * based on the idea that the geometry information is needed only once also
1302 * when several fields are evaluated, in a departure from FEValues which sets
1303 * up the internal mapping data for each field. If for example a
1304 * multi-component PDE involves the shape values on one component and the
1305 * shape gradient on the other, no efficiency is lost if both are based on the
1306 * same MatrixFree object where the update flags specify that both @p
1307 * update_values , @p update_gradients , and @p update_jxw_values are
1308 * given. The selection of desired quantities of shape values is through the
1309 * flags in the evaluate() or integrate calls and the access at quadrature
1310 * points:
1311 *
1312 * @code
1313 * fe_eval1.evaluate(EvaluationFlags::values);
1314 * fe_eval2.evaluate(EvaluationFlags::gradients);
1315 * for (unsigned int q=0; q<fe_eval1.n_q_points; ++q)
1316 * {
1317 * VectorizedArray<double> val1 = fe_eval1.get_value(q);
1318 * Tensor<1,dim,VectorizedArray<double> > grad2 = fe_eval2.get_gradient(q);
1319 * Point<dim,VectorizedArray<double> > point = fe_eval1.quadrature_point(q);
1320 * // ... some complicated formula combining those three...
1321 * }
1322 * @endcode
1323 *
1324 * In the loop over quadrature points, one can ask any of the two FEEvaluation
1325 * objects &mdash; it does not really matter which one because they only keep
1326 * pointers to the quadrature point data &mdash; to provide the quadrature
1327 * point location.
1328 *
1329 * This observation also translates to the case when different differential
1330 * operators are implemented in a program, for example the action of a mass
1331 * matrix for one phase of the algorithm and the action of a
1332 * @ref GlossStiffnessMatrix "stiffness matrix"
1333 * in another one. Only a single MatrixFree object is necessary, maintaining
1334 * full efficiency by using different local functions with the respective
1335 * implementation in separate FEEvaluation objects. In other words, a user
1336 * does not need to bother about being conservative when providing
1337 * update_flags to the initialization of MatrixFree for efficiency reasons -
1338 * no overhead incurs inside FEEvaluation, except for at most one or two more
1339 * @p if statements inside the FEEvaluation::reinit() call. Rather, the
1340 * largest set of flags necessary among all calls is perfectly fine from an
1341 * efficiency point of view.
1342 *
1343 * For the combination of different fields, including different solution
1344 * vectors that come from different time steps, it is mandatory that all
1345 * FEEvaluation objects share the same MatrixFree object. This is because the
1346 * way cells are looped by MatrixFree::cell_loop() can be different for
1347 * different DoFHandler or AffineConstraints arguments. More precisely, even
1348 * though the layout is going to be the same in serial, there is no guarantee
1349 * about the ordering for different DoFHandler/AffineConstraints in the MPI
1350 * case. The reason is that the algorithm detects cells that need data
1351 * exchange with MPI and those can change for different elements &mdash; FE_Q
1352 * with hanging node constraints connects to more neighbors than a FE_DGQ
1353 * element, for instance, and cells which need data exchange are put in
1354 * different positions inside the cell loop. Of course, if the exact same
1355 * DoFHandler, AffineConstraints, and options (such as the setting for thread
1356 * parallelism) are set, then the order is going to be the same because the
1357 * algorithm is deterministic.
1358 *
1359 * @tparam dim Dimension in which this class is to be used
1360 *
1361 * @tparam fe_degree Degree of the tensor product finite element with
1362 * fe_degree+1 degrees of freedom per coordinate direction. Can be set to -1
1363 * if the degree is not known at compile time, but performance will usually be
1364 * worse by a factor of 2-3.
1365 *
1366 * @tparam n_q_points_1d Number of points in the quadrature formula in 1d,
1367 * defaults to fe_degree+1
1368 *
1369 * @tparam n_components Number of vector components when solving a system of
1370 * PDEs. If the same operation is applied to several components of a PDE (e.g.
1371 * a vector Laplace equation), they can be applied simultaneously with one
1372 * call (and often more efficiently). Defaults to 1.
1373 *
1374 * @tparam Number Number format, usually @p double or @p float. Defaults to @p
1375 * double
1376 *
1377 * @ingroup matrixfree
1378 */
1379 template <int dim,
1380 int fe_degree,
1381 int n_q_points_1d,
1382 int n_components_,
1383 typename Number,
1384 typename VectorizedArrayType>
1385 class FEEvaluation : public FEEvaluationBase<dim,
1386 n_components_,
1387 Number,
1388 false,
1389 VectorizedArrayType>
1390 {
1391 static_assert(
1392 std::is_same_v<Number, typename VectorizedArrayType::value_type>,
1393 "Type of Number and of VectorizedArrayType do not match.");
1394
1395 public:
1396 /**
1397 * An alias to the base class.
1398 */
1399 using BaseClass =
1400 FEEvaluationBase<dim, n_components_, Number, false, VectorizedArrayType>;
1401
1402 /**
1403 * An underlying number type specified as template argument.
1404 */
1405 using number_type = Number;
1406
1407 /**
1408 * The type of function values, e.g. `VectorizedArrayType` for
1409 * `n_components=1` or `Tensor<1,dim,VectorizedArrayType >` for
1410 * `n_components=dim`.
1411 */
1412 using value_type = typename BaseClass::value_type;
1413
1414 /**
1415 * The type of gradients, e.g. `Tensor<1,dim,VectorizedArrayType>` for
1416 * `n_components=1` or `Tensor<2,dim,VectorizedArrayType >` for
1417 * `n_components=dim`.
1418 */
1419 using gradient_type = typename BaseClass::gradient_type;
1420
1421 /**
1422 * The dimension given as template argument.
1423 */
1424 static constexpr unsigned int dimension = dim;
1425
1426 /**
1427 * The number of solution components of the evaluator given as template
1428 * argument.
1429 */
1430 static constexpr unsigned int n_components = n_components_;
1431
1432 /**
1433 * The number of lanes of the template argument VectorizedArrayType.
1434 */
1435 static constexpr unsigned int n_lanes = VectorizedArrayType::size();
1436
1437 /**
1438 * The static number of quadrature points determined from the given template
1439 * argument `n_q_points_1d`.
1440 *
1441 * @note The actual number of quadrature
1442 * points, `n_q_points`, can be different if `fe_degree=-1` is given and
1443 * run-time loop lengths are used rather than compile time ones.
1444 */
1445 static constexpr unsigned int static_n_q_points =
1446 Utilities::pow(n_q_points_1d, dim);
1447
1448 /**
1449 * The static number of degrees of freedom of a scalar component determined
1450 * from the given template argument `fe_degree`.
1451 *
1452 * @note The actual number
1453 * of degrees of freedom `dofs_per_component` can be different if
1454 * `fe_degree=-1` is given or if the underlying is of more complicated type
1455 * than the usual FE_Q or FE_DGQ ones, such as FE_DGP.
1456 */
1457 static constexpr unsigned int static_dofs_per_component =
1458 Utilities::pow(fe_degree + 1, dim);
1459
1460 /**
1461 * The static number of degrees of freedom of all components determined from
1462 * the given template argument `fe_degree`.
1463 *
1464 * @note The actual number of
1465 * degrees of freedom `dofs_per_cell` can be different if `fe_degree=-1` is
1466 * given or if the underlying is of more complicated type than the usual
1467 * FE_Q or FE_DGQ ones, such as FE_DGP.
1468 */
1469 static constexpr unsigned int tensor_dofs_per_cell =
1470 static_dofs_per_component * n_components;
1471
1472 /**
1473 * The static number of degrees of freedom of all components determined from
1474 * the given template argument `fe_degree`.
1475 *
1476 * @note The actual number of
1477 * degrees of freedom `dofs_per_cell` can be different if `fe_degree=-1` is
1478 * given or if the underlying is of more complicated type than the usual
1479 * FE_Q or FE_DGQ ones, such as FE_DGP.
1480 */
1481 static constexpr unsigned int static_dofs_per_cell =
1482 static_dofs_per_component * n_components;
1483
1484 /**
1485 * Constructor. Takes all data stored in MatrixFree. If applied to problems
1486 * with more than one finite element or more than one quadrature formula
1487 * selected during construction of @p matrix_free, the appropriate component
1488 * can be selected by the optional arguments.
1489 *
1490 * @param matrix_free Data object that contains all data
1491 *
1492 * @param dof_no If matrix_free was set up with multiple DoFHandler
1493 * objects, this parameter selects to which DoFHandler/AffineConstraints pair
1494 * the given evaluator should be attached to.
1495 *
1496 * @param quad_no If matrix_free was set up with multiple Quadrature
1497 * objects, this parameter selects the appropriate number of the quadrature
1498 * formula.
1499 *
1500 * @param first_selected_component If the dof_handler selected by dof_no
1501 * uses an FESystem consisting of more than one component, this parameter
1502 * allows for selecting the component where the current evaluation routine
1503 * should start. Note that one evaluator does not support combining
1504 * different shape functions in different components. In other words, the
1505 * same base element of a FESystem needs to be set for the components
1506 * between @p first_selected_component and
1507 * <code>first_selected_component+n_components_</code>.
1508 *
1509 * @param active_fe_index If matrix_free was set up with DoFHandler
1510 * objects with hp::FECollections, this parameter selects to which
1511 * DoFHandler/AffineConstraints pair the given evaluator should be attached
1512 * to.
1513 *
1514 * @param active_quad_index If matrix_free was set up with hp::Collection
1515 * objects, this parameter selects the appropriate number of the quadrature
1516 * formula.
1517 */
1518 FEEvaluation(
1519 const MatrixFree<dim, Number, VectorizedArrayType> &matrix_free,
1520 const unsigned int dof_no = 0,
1521 const unsigned int quad_no = 0,
1522 const unsigned int first_selected_component = 0,
1523 const unsigned int active_fe_index = numbers::invalid_unsigned_int,
1524 const unsigned int active_quad_index = numbers::invalid_unsigned_int);
1525
1526 /**
1527 * Constructor. Takes all data stored in MatrixFree for a given cell range,
1528 * which allows to automatically identify the active_fe_index and
1529 * active_quad_index in case of a p-adaptive strategy.
1530 *
1531 * The rest of the arguments are the same as in the constructor above.
1532 */
1533 FEEvaluation(const MatrixFree<dim, Number, VectorizedArrayType> &matrix_free,
1534 const std::pair<unsigned int, unsigned int> &range,
1535 const unsigned int dof_no = 0,
1536 const unsigned int quad_no = 0,
1537 const unsigned int first_selected_component = 0);
1538
1539 /**
1540 * Constructor that comes with reduced functionality and works similar as
1541 * FEValues. The arguments are similar to the ones passed to the constructor
1542 * of FEValues, with the notable difference that FEEvaluation expects a
1543 * one-dimensional
1544 * quadrature formula, Quadrature<1>, instead of a @p dim
1545 * dimensional one. The finite element can be both scalar or vector valued,
1546 * but this method always only selects a scalar base element at a time (with
1547 * @p n_components copies as specified by the class template). For
1548 * vector-valued
1549 * elements, the optional argument @p first_selected_component allows
1550 * to specify the index of the base element to be used for evaluation. Note
1551 * that the internal data structures always assume that the base element is
1552 * primitive, non-primitive are not supported currently.
1553 *
1554 * As known from FEValues, a call to the reinit method with a
1555 * Triangulation<dim>::cell_iterator is necessary to make the geometry and
1556 * degrees of freedom of the current class known. If the iterator includes
1557 * DoFHandler information (i.e., it is a DoFHandler<dim>::cell_iterator or
1558 * similar), the initialization allows to also read from or write to vectors
1559 * in the standard way for DoFHandler<dim>::active_cell_iterator types for
1560 * one cell at a time. However, this approach is much slower than the path
1561 * with MatrixFree with MPI since index translation has to be done. As only
1562 * one cell at a time is used, this method does not vectorize over several
1563 * elements (which is most efficient for vector operations), but only
1564 * possibly within the element if the evaluate/integrate routines are
1565 * combined inside user code (e.g. for computing cell matrices).
1566 */
1567 FEEvaluation(const Mapping<dim> &mapping,
1568 const FiniteElement<dim> &fe,
1569 const Quadrature<1> &quadrature,
1570 const UpdateFlags update_flags,
1571 const unsigned int first_selected_component = 0);
1572
1573 /**
1574 * Constructor for the reduced functionality. This constructor is equivalent
1575 * to the other one except that it makes the object use a $Q_1$ mapping
1576 * (i.e., an object of type MappingQ(1)) implicitly.
1577 */
1578 FEEvaluation(const FiniteElement<dim> &fe,
1579 const Quadrature<1> &quadrature,
1580 const UpdateFlags update_flags,
1581 const unsigned int first_selected_component = 0);
1582
1583 /**
1584 * Constructor for the reduced functionality. Similar to the other
1585 * constructor with FiniteElement argument but using another
1586 * FEEvaluationBase object to provide information about the geometry. This
1587 * allows several FEEvaluation objects to share the geometry evaluation, i.e.,
1588 * the underlying mapping and quadrature points do only need to be evaluated
1589 * once. Make sure to not pass an optional object around when you intend to
1590 * use the FEEvaluation object in %parallel to the given one because
1591 * otherwise the intended sharing may create race conditions.
1592 */
1593 FEEvaluation(const FiniteElement<dim> &fe,
1594 const FEEvaluationData<dim, VectorizedArrayType, false> &other,
1595 const unsigned int first_selected_component = 0);
1596
1597 /**
1598 * Copy constructor. If FEEvaluationBase was constructed from a mapping, fe,
1599 * quadrature, and update flags, the underlying geometry evaluation based on
1600 * FEValues will be deep-copied in order to allow for using in parallel with
1601 * threads.
1602 */
1603 FEEvaluation(const FEEvaluation &other);
1604
1605 /**
1606 * Copy assignment operator. If FEEvaluationBase was constructed from a
1607 * mapping, fe, quadrature, and update flags, the underlying geometry
1608 * evaluation based on FEValues will be deep-copied in order to allow for
1609 * using in parallel with threads.
1610 */
1611 FEEvaluation &
1612 operator=(const FEEvaluation &other);
1613
1614 /**
1615 * Initialize the operation pointer to the current cell batch index. Unlike
1616 * the reinit functions taking a cell iterator as argument below and the
1617 * FEValues::reinit() methods, where the information related to a particular
1618 * cell is generated in the reinit call, this function is very cheap since
1619 * all data is pre-computed in @p matrix_free, and only a few indices have
1620 * to be set appropriately.
1621 */
1622 void
1623 reinit(const unsigned int cell_batch_index);
1624
1625 /**
1626 * Similar as the above function but allowing to define customized cell
1627 * batches on the fly. A cell batch is defined by the (matrix-free) index of
1628 * its cells: see also the documentation of get_cell_ids () or
1629 * get_cell_or_face_ids ().
1630 */
1631 void
1632 reinit(const std::array<unsigned int, n_lanes> &cell_ids);
1633
1634 /**
1635 * Initialize the data to the current cell using a TriaIterator object as
1636 * usual in FEValues. The argument is either of type
1637 * DoFHandler::active_cell_iterator or DoFHandler::level_cell_iterator. This
1638 * option is only available if the FEEvaluation object was created with a
1639 * finite element, quadrature formula and correct update flags and
1640 * <b>without</b> a MatrixFree object. This initialization method loses the
1641 * ability to use vectorization, see also the description of the
1642 * FEEvaluation class. When this reinit method is used, FEEvaluation can
1643 * also read from vectors (but less efficient than with data coming from
1644 * MatrixFree).
1645 */
1646 template <bool level_dof_access>
1647 void
1648 reinit(const TriaIterator<DoFCellAccessor<dim, dim, level_dof_access>> &cell);
1649
1650 /**
1651 * Initialize the data to the current cell using a TriaIterator object as
1652 * usual in FEValues. This option is only available if the FEEvaluation
1653 * object was created with a finite element, quadrature formula and correct
1654 * update flags and <b>without</b> a MatrixFree object. This initialization
1655 * method loses the ability to use vectorization, see also the description
1656 * of the FEEvaluation class. When this reinit method is used, FEEvaluation
1657 * can <b>not</b> read from vectors because no DoFHandler information is
1658 * available.
1659 */
1660 void
1661 reinit(const typename Triangulation<dim>::cell_iterator &cell);
1662
1663 /**
1664 * Check if fast evaluation/integration is supported.
1665 */
1666 static bool
1667 fast_evaluation_supported(const unsigned int given_degree,
1668 const unsigned int given_n_q_points_1d);
1669
1670 /**
1671 * Evaluate the function values, the gradients, and the Hessians of the
1672 * polynomial interpolation from the DoF values in the input vector to the
1673 * quadrature points on the unit cell. The function arguments specify which
1674 * parts shall actually be computed. This function has to be called first so
1675 * that the access functions get_value(), get_gradient() or
1676 * get_laplacian() give useful information (unless these values have been set
1677 * manually).
1678 */
1679 void
1680 evaluate(const EvaluationFlags::EvaluationFlags evaluation_flag);
1681
1682 /**
1683 * Evaluate the function values, the gradients, and the Hessians of the
1684 * polynomial interpolation from the DoF values in the input array @p
1685 * values_array to the quadrature points on the unit cell. If multiple
1686 * components are involved in the current FEEvaluation object, the sorting
1687 * in @p values_array is such that all degrees of freedom for the first
1688 * component come first, then all degrees of freedom for the second, and so
1689 * on. The function arguments specify which parts shall actually be
1690 * computed. This function has to be called first so that the access
1691 * functions get_value(), get_gradient() or get_laplacian() give
1692 * useful information (unless these values have been set manually).
1693 */
1694 void
1695 evaluate(const VectorizedArrayType *values_array,
1696 const EvaluationFlags::EvaluationFlags evaluation_flag);
1697
1698 /**
1699 * Read from the input vector and evaluates the function values, the
1700 * gradients, and the Hessians of the polynomial interpolation of the vector
1701 * entries from @p input_vector associated with the current cell to the
1702 * quadrature points on the unit cell. The function arguments specify which
1703 * parts shall actually be computed. This function has to be called first so
1704 * that the access functions get_value(), get_gradient() or
1705 * get_laplacian() give useful information (unless these values have been set
1706 * manually).
1707 *
1708 * This call is equivalent to calling read_dof_values() followed by
1709 * evaluate(), but might internally use some additional optimizations.
1710 */
1711 template <typename VectorType>
1712 void
1713 gather_evaluate(const VectorType &input_vector,
1714 const EvaluationFlags::EvaluationFlags evaluation_flag);
1715
1716 /**
1717 * This function takes the values and/or gradients that are stored on
1718 * quadrature points, tests them by all the basis functions/gradients on the
1719 * cell and performs the cell integration. The function argument
1720 * @p integration_flag is used to control which of the submitted contributions
1721 * are used during the integration. The result is written into the internal
1722 * data field dof_values (that is usually written into the result vector by
1723 * the distribute_local_to_global() or set_dof_values() methods).
1724 */
1725 void
1726 integrate(const EvaluationFlags::EvaluationFlags integration_flag);
1727
1728 /**
1729 * This function takes the values and/or gradients that are stored on
1730 * quadrature points, tests them by all the basis functions/gradients on the
1731 * cell and performs the cell integration. The function argument
1732 * @p integration_flag is used to control which of the submitted contributions
1733 * are used during the integration. As opposed to the other integrate()
1734 * method, this
1735 * call stores the result of the testing in the given array @p values_array,
1736 * whose previous results is overwritten, rather than writing it on the
1737 * internal data structures behind begin_dof_values().
1738 */
1739 void
1740 integrate(const EvaluationFlags::EvaluationFlags integration_flag,
1741 VectorizedArrayType *values_array,
1742 const bool sum_into_values = false);
1743
1744 /**
1745 * This function takes the values and/or gradients that are stored on
1746 * quadrature points, tests them by all the basis functions/gradients on the
1747 * cell, performs the cell integration, and adds the result into the global
1748 * vector @p output_vector on the degrees of freedom associated with the
1749 * present cell index. The function argument
1750 * @p integration_flag is used to control which of the submitted contributions
1751 * are used during the integration.
1752 *
1753 * This call is equivalent to calling integrate() followed by
1754 * distribute_local_to_global(), but might internally use
1755 * some additional optimizations.
1756 */
1757 template <typename VectorType>
1758 void
1759 integrate_scatter(const EvaluationFlags::EvaluationFlags integration_flag,
1760 VectorType &output_vector);
1761
1762 /**
1763 * Return an object that can be thought of as an array containing all indices
1764 * from zero to @p dofs_per_cell. This allows to write code using
1765 * range-based for loops.
1766 */
1767 std_cxx20::ranges::iota_view<unsigned int, unsigned int>
1768 dof_indices() const;
1769
1770 /**
1771 * The number of degrees of freedom of a single component on the cell for
1772 * the underlying evaluation object. Usually close to
1773 * static_dofs_per_component, but the number depends on the actual element
1774 * selected and is thus not static.
1775 */
1776 const unsigned int dofs_per_component;
1777
1778 /**
1779 * The number of degrees of freedom on the cell accumulated over all
1780 * components in the current evaluation object. Usually close to
1781 * static_dofs_per_cell = static_dofs_per_component*n_components, but the
1782 * number depends on the actual element selected and is thus not static.
1783 */
1784 const unsigned int dofs_per_cell;
1785
1786 /**
1787 * The number of quadrature points in use. If the number of quadrature
1788 * points in 1d is given as a template, this number is simply the
1789 * <tt>dim</tt>-th power of that value. If the element degree is set to -1
1790 * (dynamic selection of element degree), the static value of quadrature
1791 * points is inaccurate and this value must be used instead.
1792 */
1793 const unsigned int n_q_points;
1794
1795 private:
1796 /**
1797 * Checks if the template arguments regarding degree of the element
1798 * corresponds to the actual element used at initialization.
1799 */
1800 void
1801 check_template_arguments(const unsigned int fe_no,
1802 const unsigned int first_selected_component);
1803 };
1804
1805
1806
1807 /**
1808 * The class that provides all functions necessary to evaluate functions at
1809 * quadrature points and face integrations. The design of the class is similar
1810 * to FEEvaluation and most of the interfaces are shared with that class, in
1811 * particular most access functions that come from the common base classes
1812 * FEEvaluationData and FEEvaluationBase. Furthermore, the relation of this
1813 * class to FEEvaluation is similar to the relation between FEValues and
1814 * FEFaceValues.
1815 *
1816 * @tparam dim Dimension in which this class is to be used
1817 *
1818 * @tparam fe_degree Degree of the tensor product finite element with
1819 * fe_degree+1 degrees of freedom per coordinate
1820 * direction. If set to -1, the degree of the underlying
1821 * element will be used, which acts as a run time constant
1822 * rather than a compile time constant that slows down the
1823 * execution.
1824 *
1825 * @tparam n_q_points_1d Number of points in the quadrature formula in 1d,
1826 * usually chosen as fe_degree+1
1827 *
1828 * @tparam n_components Number of vector components when solving a system of
1829 * PDEs. If the same operation is applied to several
1830 * components of a PDE (e.g. a vector Laplace equation), they
1831 * can be applied simultaneously with one call (and often
1832 * more efficiently)
1833 *
1834 * @tparam Number Number format, usually @p double or @p float
1835 *
1836 * @tparam VectorizedArrayType Type of array to be worked on in a vectorized
1837 * fashion, defaults to VectorizedArray<Number>
1838 *
1839 * @note Currently only VectorizedArray<Number, width> is supported as
1840 * VectorizedArrayType.
1841 */
1842 template <int dim,
1843 int fe_degree,
1844 int n_q_points_1d = fe_degree + 1,
1845 int n_components_ = 1,
1846 typename Number = double,
1847 typename VectorizedArrayType = VectorizedArray<Number>>
1848 class FEFaceEvaluation : public FEEvaluationBase<dim,
1849 n_components_,
1850 Number,
1851 true,
1852 VectorizedArrayType>
1853 {
1854 static_assert(
1855 std::is_same_v<Number, typename VectorizedArrayType::value_type>,
1856 "Type of Number and of VectorizedArrayType do not match.");
1857
1858 public:
1859 /**
1860 * An alias to the base class.
1861 */
1862 using BaseClass =
1863 FEEvaluationBase<dim, n_components_, Number, true, VectorizedArrayType>;
1864
1865 /**
1866 * A underlying number type specified as template argument.
1867 */
1868 using number_type = Number;
1869
1870 /**
1871 * The type of function values, e.g. `VectorizedArrayType` for
1872 * `n_components=1` or `Tensor<1,dim,VectorizedArrayType >` for
1873 * `n_components=dim`.
1874 */
1875 using value_type = typename BaseClass::value_type;
1876
1877 /**
1878 * The type of gradients, e.g. `Tensor<1,dim,VectorizedArrayType>` for
1879 * `n_components=1` or `Tensor<2,dim,VectorizedArrayType >` for
1880 * `n_components=dim`.
1881 */
1882 using gradient_type = typename BaseClass::gradient_type;
1883
1884 /**
1885 * The dimension given as template argument.
1886 */
1887 static constexpr unsigned int dimension = dim;
1888
1889 /**
1890 * The number of solution components of the evaluator given as template
1891 * argument.
1892 */
1893 static constexpr unsigned int n_components = n_components_;
1894
1895 /**
1896 * The number of lanes of the template argument VectorizedArrayType.
1897 */
1898 static constexpr unsigned int n_lanes = VectorizedArrayType::size();
1899
1900 /**
1901 * The static number of quadrature points determined from the given template
1902 * argument `n_q_points_1d` taken to the power of dim-1.
1903 *
1904 * @note The actual
1905 * number of quadrature points, `n_q_points`, can be different if
1906 * `fe_degree=-1` is given and run-time loop lengths are used rather than
1907 * compile time ones.
1908 */
1909 static constexpr unsigned int static_n_q_points =
1910 Utilities::pow(n_q_points_1d, dim - 1);
1911
1912 /**
1913 * The static number of quadrature points on a cell with the same quadrature
1914 * formula.
1915 *
1916 * @note This value is only present for simpler comparison with
1917 * the cell quadrature, as the actual number of points is given to a face by
1918 * the `static_n_q_points` variable.
1919 */
1920 static constexpr unsigned int static_n_q_points_cell =
1921 Utilities::pow(n_q_points_1d, dim);
1922
1923 /**
1924 * The static number of degrees of freedom of a scalar component determined
1925 * from the given template argument `fe_degree`.
1926 *
1927 * @note The actual number
1928 * of degrees of freedom `dofs_per_component` can be different if
1929 * `fe_degree=-1` is given.
1930 */
1931 static constexpr unsigned int static_dofs_per_component =
1932 Utilities::pow(fe_degree + 1, dim);
1933
1934 /**
1935 * The static number of degrees of freedom of all components determined from
1936 * the given template argument `fe_degree`.
1937 *
1938 * @note The actual number of
1939 * degrees of freedom `dofs_per_cell` can be different if `fe_degree=-1` is
1940 * given.
1941 */
1942 static constexpr unsigned int tensor_dofs_per_cell =
1943 static_dofs_per_component * n_components;
1944
1945 /**
1946 * The static number of degrees of freedom of all components determined from
1947 * the given template argument `fe_degree`.
1948 *
1949 * @note The actual number of
1950 * degrees of freedom `dofs_per_cell` can be different if `fe_degree=-1` is
1951 * given.
1952 */
1953 static constexpr unsigned int static_dofs_per_cell =
1954 static_dofs_per_component * n_components;
1955
1956 /**
1957 * Constructor. Takes all data stored in MatrixFree. If applied to problems
1958 * with more than one finite element or more than one quadrature formula
1959 * selected during construction of @p matrix_free, the appropriate component
1960 * can be selected by the optional arguments.
1961 *
1962 * @param matrix_free Data object that contains all data
1963 *
1964 * @param is_interior_face This selects which of the two cells of an
1965 * internal face the current evaluator will be based upon. The interior face
1966 * is the main face along which the normal vectors are oriented. The
1967 * exterior face coming from the other side provides the same normal vector
1968 * as the interior side, so if the outer normal vector to that side is
1969 * desired, it must be multiplied by -1.
1970 *
1971 * @param dof_no If matrix_free was set up with multiple DoFHandler
1972 * objects, this parameter selects to which DoFHandler/AffineConstraints pair
1973 * the given evaluator should be attached to.
1974 *
1975 * @param quad_no If matrix_free was set up with multiple Quadrature
1976 * objects, this parameter selects the appropriate number of the quadrature
1977 * formula.
1978 *
1979 * @param first_selected_component If the dof_handler selected by dof_no
1980 * uses an FESystem consisting of more than one base element, this parameter
1981 * selects the number of the base element in FESystem. Note that this does
1982 * not directly relate to the component of the respective element due to the
1983 * possibility for a multiplicity in the element.
1984 *
1985 * @param active_fe_index If matrix_free was set up with DoFHandler
1986 * objects with hp::FECollections, this parameter selects to which
1987 * DoFHandler/AffineConstraints pair the given evaluator should be attached
1988 * to.
1989 *
1990 * @param face_type In the case of a face, indicate its reference-cell type
1991 * (0 for line or quadrilateral 1 for triangle).
1992 *
1993 * @param active_quad_index If matrix_free was set up with hp::Collection
1994 * objects, this parameter selects the appropriate number of the quadrature
1995 * formula.
1996 */
1997 FEFaceEvaluation(
1998 const MatrixFree<dim, Number, VectorizedArrayType> &matrix_free,
1999 const bool is_interior_face = true,
2000 const unsigned int dof_no = 0,
2001 const unsigned int quad_no = 0,
2002 const unsigned int first_selected_component = 0,
2003 const unsigned int active_fe_index = numbers::invalid_unsigned_int,
2004 const unsigned int active_quad_index = numbers::invalid_unsigned_int,
2005 const unsigned int face_type = numbers::invalid_unsigned_int);
2006
2007 /**
2008 * Constructor. Takes all data stored in MatrixFree for a given face range,
2009 * which allows to automatically identify the active_fe_index and
2010 * active_quad_index in case of a p-adaptive strategy.
2011 *
2012 * The rest of the arguments are the same as in the constructor above.
2013 */
2014 FEFaceEvaluation(
2015 const MatrixFree<dim, Number, VectorizedArrayType> &matrix_free,
2016 const std::pair<unsigned int, unsigned int> &range,
2017 const bool is_interior_face = true,
2018 const unsigned int dof_no = 0,
2019 const unsigned int quad_no = 0,
2020 const unsigned int first_selected_component = 0);
2021
2022 /**
2023 * Initializes the operation pointer to the current face. This method is the
2024 * default choice for face integration as the data stored in MappingInfo is
2025 * stored according to this numbering. Unlike the reinit functions taking a
2026 * cell iterator as argument below and the FEValues::reinit() methods, where
2027 * the information related to a particular cell is generated in the reinit
2028 * call, this function is very cheap since all data is pre-computed in
2029 * @p matrix_free, and only a few indices and pointers have to be set
2030 * appropriately.
2031 */
2032 void
2033 reinit(const unsigned int face_batch_number);
2034
2035 /**
2036 * As opposed to the reinit() method from the base class, this reinit()
2037 * method initializes for a given number of cells and a face number. This
2038 * method is less efficient than the other reinit() method taking a
2039 * numbering of the faces because it needs to copy the data associated with
2040 * the faces to the cells in this call.
2041 */
2042 void
2043 reinit(const unsigned int cell_batch_number, const unsigned int face_number);
2044
2045 /**
2046 * Check if fast evaluation/integration is supported.
2047 */
2048 static bool
2049 fast_evaluation_supported(const unsigned int given_degree,
2050 const unsigned int given_n_q_points_1d);
2051
2052 /**
2053 * Evaluates the function values, the gradients, and the Laplacians of the
2054 * FE function given at the DoF values stored in the internal data field
2055 * `dof_values` (that is usually filled by the read_dof_values() method) at
2056 * the quadrature points on the unit cell. The function arguments specify
2057 * which parts shall actually be computed. Needs to be called before the
2058 * functions get_value(), get_gradient() or get_normal_derivative() give
2059 * useful information (unless these values have been set manually by
2060 * accessing the internal data pointers).
2061 */
2062 void
2063 evaluate(const EvaluationFlags::EvaluationFlags evaluation_flag);
2064
2065 /**
2066 * Evaluates the function values, the gradients, and the Laplacians of the
2067 * FE function given at the DoF values in the input array `values_array` at
2068 * the quadrature points on the unit cell. If multiple components are
2069 * involved in the current FEEvaluation object, the sorting in values_array
2070 * is such that all degrees of freedom for the first component come first,
2071 * then all degrees of freedom for the second, and so on. The function
2072 * arguments specify which parts shall actually be computed. Needs to be
2073 * called before the functions get_value(), get_gradient(), or
2074 * get_normal_derivative() give useful information (unless these values have
2075 * been set manually).
2076 */
2077 void
2078 evaluate(const VectorizedArrayType *values_array,
2079 const EvaluationFlags::EvaluationFlags evaluation_flag);
2080
2081 /**
2082 * Projects the values, the gradients, and the Hessians into the face DoFs of
2083 * the current face using the internally stored cell DoFs.
2084 */
2085 void
2086 project_to_face(const EvaluationFlags::EvaluationFlags evaluation_flag);
2087
2088 /**
2089 * Projects the values, the gradients, and the Hessians into the face DoFs of
2090 * the current face using the cell DoFs provided via `values_array`.
2091 */
2092 void
2093 project_to_face(const VectorizedArrayType *values_array,
2094 const EvaluationFlags::EvaluationFlags evaluation_flag);
2095
2096 /**
2097 * Evaluates the values, the gradients, and the Hessians in-face,
2098 * interpolating into the face quadrature points.
2099 */
2100 void
2101 evaluate_in_face(const EvaluationFlags::EvaluationFlags evaluation_flag);
2102
2103 /**
2104 * Reads from the input vector and evaluates the function values, the
2105 * gradients, and the Laplacians of the FE function at the quadrature points
2106 * on the unit cell. The function arguments specify which parts shall
2107 * actually be computed. Needs to be called before the functions
2108 * get_value(), get_gradient(), or get_normal_derivative() give useful
2109 * information.
2110 *
2111 * This call is equivalent to calling read_dof_values() followed by
2112 * evaluate(), but might internally use some additional optimizations.
2113 */
2114 template <typename VectorType>
2115 void
2116 gather_evaluate(const VectorType &input_vector,
2117 const EvaluationFlags::EvaluationFlags evaluation_flag);
2118
2119 /**
2120 * This function takes the values and/or gradients that are stored on
2121 * quadrature points, tests them by all the basis functions/gradients on the
2122 * cell and performs the cell integration. The function argument
2123 * @p integration_flag is used to control which of the submitted contributions
2124 * are used during the integration. The result is written into the internal
2125 * data field `dof_values` (that is usually written into the result vector by
2126 * the distribute_local_to_global() or set_dof_values() methods).
2127 */
2128 void
2129 integrate(const EvaluationFlags::EvaluationFlags integration_flag,
2130 const bool sum_into_values = false);
2131
2132 /**
2133 * This function takes the values and/or gradients that are stored on
2134 * quadrature points, tests them by all the basis functions/gradients on the
2135 * cell and performs the cell integration. The function argument
2136 * @p integration_flag is used to control which of the submitted contributions
2137 * are used during the integration. As opposed to the other integrate()
2138 * method, this
2139 * call stores the result of the testing in the given array @p values_array.
2140 */
2141 void
2142 integrate(const EvaluationFlags::EvaluationFlags integration_flag,
2143 VectorizedArrayType *values_array,
2144 const bool sum_into_values = false);
2145
2146 /**
2147 * This function tests the values, gradients and Hessians submitted on the
2148 * face quadrature points by multiplying with the in-face basis function
2149 * values, gradients and Hessians and accumulating to the respective face
2150 * DoFs.
2151 */
2152 void
2153 integrate_in_face(const EvaluationFlags::EvaluationFlags integration_flag);
2154
2155 /**
2156 * Collects the contributions from the face DoFs of values, normal gradients
2157 * and normal Hessians to the internal cell DoFs.
2158 */
2159 void
2160 collect_from_face(const EvaluationFlags::EvaluationFlags integration_flag,
2161 const bool sum_into_values = false);
2162
2163 /**
2164 * Collects the contributions from the face DoFs of values, normal gradients
2165 * and normal Hessians to the cell DoFs specified via `values_array`.
2166 */
2167 void
2168 collect_from_face(const EvaluationFlags::EvaluationFlags integration_flag,
2169 VectorizedArrayType *values_array,
2170 const bool sum_into_values = false);
2171
2172 /**
2173 * This function takes the values and/or gradients that are stored on
2174 * quadrature points, tests them by all the basis functions/gradients on the
2175 * cell and performs the cell integration. The function argument
2176 * @p integration_flag is used to control which of the submitted contributions
2177 * are used during the integration.
2178 *
2179 * This call is equivalent to calling integrate() followed by
2180 * distribute_local_to_global(), but might internally use some additional
2181 * optimizations.
2182 */
2183 template <typename VectorType>
2184 void
2185 integrate_scatter(const EvaluationFlags::EvaluationFlags integration_flag,
2186 VectorType &output_vector);
2187
2188 /**
2189 * @deprecated Please use the integrate_scatter() function with the EvaluationFlags argument.
2190 */
2191 template <typename VectorType>
2192 void
2193 integrate_scatter(const bool integrate_values,
2194 const bool integrate_gradients,
2195 VectorType &output_vector);
2196
2197 /**
2198 * Return an object that can be thought of as an array containing all indices
2199 * from zero to @p dofs_per_cell. This allows to write code using
2200 * range-based for loops.
2201 */
2202 std_cxx20::ranges::iota_view<unsigned int, unsigned int>
2203 dof_indices() const;
2204
2205 /**
2206 * Return whether the face associated to this FEFaceEvaluation object
2207 * is at the boundary.
2208 */
2209 bool
2210 at_boundary() const;
2211
2212 /**
2213 * Return the boundary indicator of the face associated to this
2214 * FEFaceEvaluation object.
2215 *
2216 * If the return value is the special value
2217 * numbers::internal_face_boundary_id, then the face is in the interior of
2218 * the domain.
2219 *
2220 * @note Alternatively to this function, you can use
2221 * MatrixFree::get_boundary_id() to get the same information if no
2222 * FEFaceEvaluation object has been set up.
2223 */
2224 types::boundary_id
2225 boundary_id() const;
2226
2227 /**
2228 * Get the number of degrees of freedom of a single component which are
2229 * projected onto a face.
2230 */
2231 unsigned int
2232 get_dofs_per_component_projected_to_face();
2233
2234 /**
2235 * Get the number of degrees of freedom accumulated over all
2236 * components which are projected onto a face.
2237 */
2238 unsigned int
2239 get_dofs_projected_to_face();
2240
2241 /**
2242 * The number of degrees of freedom of a single component on the cell for
2243 * the underlying evaluation object. Usually close to
2244 * static_dofs_per_component, but the number depends on the actual element
2245 * selected and is thus not static.
2246 */
2247 const unsigned int dofs_per_component;
2248
2249 /**
2250 * The number of degrees of freedom on the cell accumulated over all
2251 * components in the current evaluation object. Usually close to
2252 * static_dofs_per_cell = static_dofs_per_component*n_components, but the
2253 * number depends on the actual element selected and is thus not static.
2254 */
2255 const unsigned int dofs_per_cell;
2256
2257 /**
2258 * The number of quadrature points in use. If the number of quadrature
2259 * points in 1d is given as a template, this number is simply the
2260 * <tt>dim-1</tt>-th power of that value. If the element degree is set to -1
2261 * (dynamic selection of element degree), the static value of quadrature
2262 * points is inaccurate and this value must be used instead.
2263 */
2264 const unsigned int n_q_points;
2265 };
2266
2267
2268
2269 namespace internal
2270 {
2271 namespace MatrixFreeFunctions
2272 {
2273 // a helper function to compute the number of DoFs of a DGP element at
2274 // compile time, depending on the degree
2275 template <int dim, int degree>
2276 struct DGP_dofs_per_component
2277 {
2278 // this division is always without remainder
2279 static constexpr unsigned int value =
2280 (DGP_dofs_per_component<dim - 1, degree>::value * (degree + dim)) / dim;
2281 };
2282
2283 // base specialization: 1d elements have 'degree+1' degrees of freedom
2284 template <int degree>
2285 struct DGP_dofs_per_component<1, degree>
2286 {
2287 static constexpr unsigned int value = degree + 1;
2288 };
2289 } // namespace MatrixFreeFunctions
2290 } // namespace internal
2291
2292
2293 /*----------------------- Inline functions ----------------------------------*/
2294
2295 #ifndef DOXYGEN
2296
2297
2298 namespace internal
2299 {
2300 // Extract all internal data pointers and indices in a single function that
2301 // get passed on to the constructor of FEEvaluationData, avoiding to look
2302 // things up multiple times
2303 template <bool is_face,
2304 int dim,
2305 typename Number,
2306 typename VectorizedArrayType>
2307 inline typename FEEvaluationData<dim, VectorizedArrayType, is_face>::
2308 InitializationData
2309 extract_initialization_data(
2310 const MatrixFree<dim, Number, VectorizedArrayType> &matrix_free,
2311 const unsigned int dof_no,
2312 const unsigned int first_selected_component,
2313 const unsigned int quad_no,
2314 const unsigned int fe_degree,
2315 const unsigned int n_q_points,
2316 const unsigned int active_fe_index_given,
2317 const unsigned int active_quad_index_given,
2318 const unsigned int face_type)
2319 {
2320 typename FEEvaluationData<dim, VectorizedArrayType, is_face>::
2321 InitializationData init_data;
2322
2323 init_data.dof_info = &matrix_free.get_dof_info(dof_no);
2324 init_data.mapping_data =
2325 &internal::MatrixFreeFunctions::
2326 MappingInfoCellsOrFaces<dim, Number, is_face, VectorizedArrayType>::get(
2327 matrix_free.get_mapping_info(), quad_no);
2328
2329 init_data.active_fe_index =
2330 fe_degree != numbers::invalid_unsigned_int ?
2331 init_data.dof_info->fe_index_from_degree(first_selected_component,
2332 fe_degree) :
2333 (active_fe_index_given != numbers::invalid_unsigned_int ?
2334 active_fe_index_given :
2335 0);
2336
2337 init_data.active_quad_index =
2338 fe_degree == numbers::invalid_unsigned_int ?
2339 (active_quad_index_given != numbers::invalid_unsigned_int ?
2340 active_quad_index_given :
2341 std::min<unsigned int>(
2342 init_data.active_fe_index,
2343 init_data.mapping_data->descriptor.size() /
2344 (is_face ? std::max<unsigned int>(1, dim - 1) : 1) -
2345 1)) :
2346 init_data.mapping_data->quad_index_from_n_q_points(n_q_points);
2347
2348 init_data.shape_info = &matrix_free.get_shape_info(
2349 dof_no,
2350 quad_no,
2351 init_data.dof_info->component_to_base_index[first_selected_component],
2352 init_data.active_fe_index,
2353 init_data.active_quad_index);
2354 init_data.descriptor =
2355 &init_data.mapping_data->descriptor
2356 [is_face ?
2357 (init_data.active_quad_index * std::max<unsigned int>(1, dim - 1) +
2358 (face_type == numbers::invalid_unsigned_int ? 0 : face_type)) :
2359 init_data.active_quad_index];
2360
2361 return init_data;
2362 }
2363 } // namespace internal
2364
2365
2366
2367 /*----------------------- FEEvaluationBase ----------------------------------*/
2368
2369 template <int dim,
2370 int n_components_,
2371 typename Number,
2372 bool is_face,
2373 typename VectorizedArrayType>
2374 inline FEEvaluationBase<dim,
2375 n_components_,
2376 Number,
2377 is_face,
2378 VectorizedArrayType>::
2379 FEEvaluationBase(
2380 const MatrixFree<dim, Number, VectorizedArrayType> &matrix_free,
2381 const unsigned int dof_no,
2382 const unsigned int first_selected_component,
2383 const unsigned int quad_no,
2384 const unsigned int fe_degree,
2385 const unsigned int n_q_points,
2386 const bool is_interior_face,
2387 const unsigned int active_fe_index,
2388 const unsigned int active_quad_index,
2389 const unsigned int face_type)
2390 : FEEvaluationData<dim, VectorizedArrayType, is_face>(
2391 internal::extract_initialization_data<is_face>(matrix_free,
2392 dof_no,
2393 first_selected_component,
2394 quad_no,
2395 fe_degree,
2396 n_q_points,
2397 active_fe_index,
2398 active_quad_index,
2399 face_type),
2400 is_interior_face,
2401 quad_no,
2402 first_selected_component)
2403 , scratch_data_array(matrix_free.acquire_scratch_data())
2404 , matrix_free(&matrix_free)
2405 {
2406 this->set_data_pointers(scratch_data_array, n_components_);
2407 Assert(
2408 this->dof_info->start_components.back() == 1 ||
2409 static_cast<int>(n_components_) <=
2410 static_cast<int>(
2411 this->dof_info->start_components
2412 [this->dof_info->component_to_base_index[first_selected_component] +
2413 1]) -
2414 first_selected_component,
2415 ExcMessage(
2416 "You tried to construct a vector-valued evaluator with " +
2417 std::to_string(n_components) +
2418 " components. However, "
2419 "the current base element has only " +
2420 std::to_string(
2421 this->dof_info->start_components
2422 [this->dof_info->component_to_base_index[first_selected_component] +
2423 1] -
2424 first_selected_component) +
2425 " components left when starting from local element index " +
2426 std::to_string(
2427 first_selected_component -
2428 this->dof_info->start_components
2429 [this->dof_info->component_to_base_index[first_selected_component]]) +
2430 " (global index " + std::to_string(first_selected_component) + ")"));
2431
2432 // do not check for correct dimensions of data fields here, should be done
2433 // in derived classes
2434 }
2435
2436
2437
2438 template <int dim,
2439 int n_components_,
2440 typename Number,
2441 bool is_face,
2442 typename VectorizedArrayType>
2443 inline FEEvaluationBase<dim,
2444 n_components_,
2445 Number,
2446 is_face,
2447 VectorizedArrayType>::
2448 FEEvaluationBase(
2449 const Mapping<dim> &mapping,
2450 const FiniteElement<dim> &fe,
2451 const Quadrature<1> &quadrature,
2452 const UpdateFlags update_flags,
2453 const unsigned int first_selected_component,
2454 const FEEvaluationData<dim, VectorizedArrayType, is_face> *other)
2455 : FEEvaluationData<dim, VectorizedArrayType, is_face>(
2456 other != nullptr &&
2457 other->mapped_geometry->get_quadrature() == quadrature ?
2458 other->mapped_geometry :
2459 std::make_shared<internal::MatrixFreeFunctions::
2460 MappingDataOnTheFly<dim, VectorizedArrayType>>(
2461 mapping,
2462 quadrature,
2463 update_flags),
2464 n_components_,
2465 first_selected_component)
2466 , scratch_data_array(new AlignedVector<VectorizedArrayType>())
2467 , matrix_free(nullptr)
2468 {
2469 const unsigned int base_element_number =
2470 fe.component_to_base_index(first_selected_component).first;
2471 Assert(fe.element_multiplicity(base_element_number) == 1 ||
2472 fe.element_multiplicity(base_element_number) -
2473 first_selected_component >=
2474 n_components_,
2475 ExcMessage("The underlying element must at least contain as many "
2476 "components as requested by this class"));
2477 (void)base_element_number;
2478
2479 Assert(this->data == nullptr, ExcInternalError());
2480 this->data = new internal::MatrixFreeFunctions::ShapeInfo<Number>(
2481 Quadrature<(is_face ? dim - 1 : dim)>(quadrature),
2482 fe,
2483 fe.component_to_base_index(first_selected_component).first);
2484
2485 this->set_data_pointers(scratch_data_array, n_components_);
2486 }
2487
2488
2489
2490 template <int dim,
2491 int n_components_,
2492 typename Number,
2493 bool is_face,
2494 typename VectorizedArrayType>
2495 inline FEEvaluationBase<dim,
2496 n_components_,
2497 Number,
2498 is_face,
2499 VectorizedArrayType>::
2500 FEEvaluationBase(const FEEvaluationBase<dim,
2501 n_components_,
2502 Number,
2503 is_face,
2504 VectorizedArrayType> &other)
2505 : FEEvaluationData<dim, VectorizedArrayType, is_face>(other)
2506 , scratch_data_array(other.matrix_free == nullptr ?
2507 new AlignedVector<VectorizedArrayType>() :
2508 other.matrix_free->acquire_scratch_data())
2509 , matrix_free(other.matrix_free)
2510 {
2511 if (other.matrix_free == nullptr)
2512 {
2513 Assert(other.mapped_geometry.get() != nullptr, ExcInternalError());
2514 this->data =
2515 new internal::MatrixFreeFunctions::ShapeInfo<Number>(*other.data);
2516
2517 // Create deep copy of mapped geometry for use in parallel
2518 this->mapped_geometry =
2519 std::make_shared<internal::MatrixFreeFunctions::
2520 MappingDataOnTheFly<dim, VectorizedArrayType>>(
2521 other.mapped_geometry->get_fe_values().get_mapping(),
2522 other.mapped_geometry->get_quadrature(),
2523 other.mapped_geometry->get_fe_values().get_update_flags());
2524
2525 if constexpr (is_face == false)
2526 this->mapping_data = &this->mapped_geometry->get_data_storage();
2527 else
2528 Assert(false,
2529 ExcNotImplemented("On-the-fly geometry evaluation with "
2530 "face evaluators is not currently "
2531 "implemented!"));
2532
2533 this->cell = 0;
2534
2535 this->jacobian =
2536 this->mapped_geometry->get_data_storage().jacobians[0].begin();
2537 this->J_value =
2538 this->mapped_geometry->get_data_storage().JxW_values.begin();
2539 this->jacobian_gradients =
2540 this->mapped_geometry->get_data_storage().jacobian_gradients[0].begin();
2541 this->jacobian_gradients_non_inverse =
2542 this->mapped_geometry->get_data_storage()
2543 .jacobian_gradients_non_inverse[0]
2544 .begin();
2545 this->quadrature_points =
2546 this->mapped_geometry->get_data_storage().quadrature_points.begin();
2547 }
2548
2549 this->set_data_pointers(scratch_data_array, n_components_);
2550 }
2551
2552
2553
2554 template <int dim,
2555 int n_components_,
2556 typename Number,
2557 bool is_face,
2558 typename VectorizedArrayType>
2559 inline FEEvaluationBase<dim,
2560 n_components_,
2561 Number,
2562 is_face,
2563 VectorizedArrayType> &
2564 FEEvaluationBase<dim, n_components_, Number, is_face, VectorizedArrayType>::
2565 operator=(const FEEvaluationBase<dim,
2566 n_components_,
2567 Number,
2568 is_face,
2569 VectorizedArrayType> &other)
2570 {
2571 // release old memory
2572 if (matrix_free == nullptr)
2573 {
2574 delete this->data;
2575 delete scratch_data_array;
2576 }
2577 else
2578 {
2579 matrix_free->release_scratch_data(scratch_data_array);
2580 }
2581
2582 this->FEEvaluationData<dim, VectorizedArrayType, is_face>::operator=(other);
2583
2584 matrix_free = other.matrix_free;
2585
2586 if (other.matrix_free == nullptr)
2587 {
2588 Assert(other.mapped_geometry.get() != nullptr, ExcInternalError());
2589 this->data =
2590 new internal::MatrixFreeFunctions::ShapeInfo<Number>(*other.data);
2591 scratch_data_array = new AlignedVector<VectorizedArrayType>();
2592
2593 // Create deep copy of mapped geometry for use in parallel
2594 this->mapped_geometry =
2595 std::make_shared<internal::MatrixFreeFunctions::
2596 MappingDataOnTheFly<dim, VectorizedArrayType>>(
2597 other.mapped_geometry->get_fe_values().get_mapping(),
2598 other.mapped_geometry->get_quadrature(),
2599 other.mapped_geometry->get_fe_values().get_update_flags());
2600
2601 if constexpr (is_face == false)
2602 this->mapping_data = &this->mapped_geometry->get_data_storage();
2603 else
2604 Assert(false,
2605 ExcNotImplemented("On-the-fly geometry evaluation with "
2606 "face evaluators is not currently "
2607 "implemented!"));
2608 this->cell = 0;
2609
2610 this->jacobian =
2611 this->mapped_geometry->get_data_storage().jacobians[0].begin();
2612 this->J_value =
2613 this->mapped_geometry->get_data_storage().JxW_values.begin();
2614 this->jacobian_gradients =
2615 this->mapped_geometry->get_data_storage().jacobian_gradients[0].begin();
2616 this->jacobian_gradients_non_inverse =
2617 this->mapped_geometry->get_data_storage()
2618 .jacobian_gradients_non_inverse[0]
2619 .begin();
2620 this->quadrature_points =
2621 this->mapped_geometry->get_data_storage().quadrature_points.begin();
2622 }
2623 else
2624 {
2625 scratch_data_array = matrix_free->acquire_scratch_data();
2626 }
2627
2628 this->set_data_pointers(scratch_data_array, n_components_);
2629
2630 return *this;
2631 }
2632
2633
2634
2635 template <int dim,
2636 int n_components_,
2637 typename Number,
2638 bool is_face,
2639 typename VectorizedArrayType>
2640 inline FEEvaluationBase<dim,
2641 n_components_,
2642 Number,
2643 is_face,
2644 VectorizedArrayType>::~FEEvaluationBase()
2645 {
2646 if (matrix_free != nullptr)
2647 {
2648 try
2649 {
2650 matrix_free->release_scratch_data(scratch_data_array);
2651 }
2652 catch (...)
2653 {}
2654 }
2655 else
2656 {
2657 delete scratch_data_array;
2658 delete this->data;
2659 }
2660 }
2661
2662
2663
2664 template <int dim,
2665 int n_components_,
2666 typename Number,
2667 bool is_face,
2668 typename VectorizedArrayType>
2669 inline const MatrixFree<dim, Number, VectorizedArrayType> &
2670 FEEvaluationBase<dim, n_components_, Number, is_face, VectorizedArrayType>::
2671 get_matrix_free() const
2672 {
2673 Assert(matrix_free != nullptr,
2674 ExcMessage(
2675 "FEEvaluation was not initialized with a MatrixFree object!"));
2676 return *matrix_free;
2677 }
2678
2679
2680
2681 namespace internal
2682 {
2683 // given a block vector return the underlying vector type
2684 // including constness (specified by bool)
2685 template <typename VectorType, bool>
2686 struct ConstBlockVectorSelector;
2687
2688 template <typename VectorType>
2689 struct ConstBlockVectorSelector<VectorType, true>
2690 {
2691 using BaseVectorType = const typename VectorType::BlockType;
2692 };
2693
2694 template <typename VectorType>
2695 struct ConstBlockVectorSelector<VectorType, false>
2696 {
2697 using BaseVectorType = typename VectorType::BlockType;
2698 };
2699
2700 // allows to select between block vectors and non-block vectors, which
2701 // allows to use a unified interface for extracting blocks on block vectors
2702 // and doing nothing on usual vectors
2703 template <typename VectorType, bool>
2704 struct BlockVectorSelector;
2705
2706 template <typename VectorType>
2707 struct BlockVectorSelector<VectorType, true>
2708 {
2709 using BaseVectorType = typename ConstBlockVectorSelector<
2710 VectorType,
2711 std::is_const_v<VectorType>>::BaseVectorType;
2712
2713 static BaseVectorType *
2714 get_vector_component(VectorType &vec, const unsigned int component)
2715 {
2716 AssertIndexRange(component, vec.n_blocks());
2717 return &vec.block(component);
2718 }
2719 };
2720
2721 template <typename VectorType>
2722 struct BlockVectorSelector<VectorType, false>
2723 {
2724 using BaseVectorType = VectorType;
2725
2726 static BaseVectorType *
2727 get_vector_component(VectorType &vec, const unsigned int component)
2728 {
2729 // FEEvaluation allows to combine several vectors from a scalar
2730 // FiniteElement into a "vector-valued" FEEvaluation object with
2731 // multiple components. These components can be extracted with the other
2732 // get_vector_component functions. If we do not get a vector of vectors
2733 // (std::vector<VectorType>, std::vector<VectorType*>, BlockVector), we
2734 // must make sure that we do not duplicate the components in input
2735 // and/or duplicate the resulting integrals. In such a case, we should
2736 // only get the zeroth component in the vector contained set nullptr for
2737 // the others which allows us to catch unintended use in
2738 // read_write_operation.
2739 if (component == 0)
2740 return &vec;
2741 else
2742 return nullptr;
2743 }
2744 };
2745
2746 template <typename VectorType>
2747 struct BlockVectorSelector<std::vector<VectorType>, false>
2748 {
2749 using BaseVectorType = VectorType;
2750
2751 static BaseVectorType *
2752 get_vector_component(std::vector<VectorType> &vec,
2753 const unsigned int component)
2754 {
2755 AssertIndexRange(component, vec.size());
2756 return &vec[component];
2757 }
2758 };
2759
2760 template <typename VectorType>
2761 struct BlockVectorSelector<const std::vector<VectorType>, false>
2762 {
2763 using BaseVectorType = const VectorType;
2764
2765 static const BaseVectorType *
2766 get_vector_component(const std::vector<VectorType> &vec,
2767 const unsigned int component)
2768 {
2769 AssertIndexRange(component, vec.size());
2770 return &vec[component];
2771 }
2772 };
2773
2774 template <typename VectorType>
2775 struct BlockVectorSelector<std::vector<VectorType *>, false>
2776 {
2777 using BaseVectorType = VectorType;
2778
2779 static BaseVectorType *
2780 get_vector_component(std::vector<VectorType *> &vec,
2781 const unsigned int component)
2782 {
2783 AssertIndexRange(component, vec.size());
2784 return vec[component];
2785 }
2786 };
2787
2788 template <typename VectorType>
2789 struct BlockVectorSelector<const std::vector<VectorType *>, false>
2790 {
2791 using BaseVectorType = const VectorType;
2792
2793 static const BaseVectorType *
2794 get_vector_component(const std::vector<VectorType *> &vec,
2795 const unsigned int component)
2796 {
2797 AssertIndexRange(component, vec.size());
2798 return vec[component];
2799 }
2800 };
2801
2802 template <typename VectorType, std::size_t N>
2803 struct BlockVectorSelector<std::array<VectorType *, N>, false>
2804 {
2805 using BaseVectorType = VectorType;
2806
2807 static BaseVectorType *
2808 get_vector_component(std::array<VectorType *, N> &vec,
2809 const unsigned int component)
2810 {
2811 AssertIndexRange(component, vec.size());
2812 return vec[component];
2813 }
2814 };
2815 } // namespace internal
2816
2817
2818
2819 template <int dim,
2820 int n_components_,
2821 typename Number,
2822 bool is_face,
2823 typename VectorizedArrayType>
2824 template <typename VectorType, typename VectorOperation>
2825 inline void
2826 FEEvaluationBase<dim, n_components_, Number, is_face, VectorizedArrayType>::
2827 read_write_operation(
2828 const VectorOperation &operation,
2829 const std::array<VectorType *, n_components_> &src,
2830 const std::array<
2831 const std::vector<ArrayView<const typename VectorType::value_type>> *,
2832 n_components_> &src_sm,
2833 const std::bitset<n_lanes> &mask,
2834 const bool apply_constraints) const
2835 {
2836 // Case 1: No MatrixFree object given, simple case because we do not need to
2837 // process constraints and need not care about vectorization -> go to
2838 // separate function
2839 if (this->matrix_free == nullptr)
2840 {
2841 read_write_operation_global(operation, src);
2842 return;
2843 }
2844
2845 Assert(this->dof_info != nullptr, ExcNotInitialized());
2846 const internal::MatrixFreeFunctions::DoFInfo &dof_info = *this->dof_info;
2847 Assert(this->matrix_free->indices_initialized() == true, ExcNotInitialized());
2848 if (this->n_fe_components == 1)
2849 for (unsigned int comp = 0; comp < n_components; ++comp)
2850 {
2851 Assert(src[comp] != nullptr,
2852 ExcMessage("The finite element underlying this FEEvaluation "
2853 "object is scalar, but you requested " +
2854 std::to_string(n_components) +
2855 " components via the template argument in "
2856 "FEEvaluation. In that case, you must pass an "
2857 "std::vector<VectorType> or a BlockVector to " +
2858 "read_dof_values and distribute_local_to_global."));
2859 internal::check_vector_compatibility(*src[comp],
2860 *this->matrix_free,
2861 *this->dof_info);
2862 }
2863 else
2864 {
2865 internal::check_vector_compatibility(*src[0],
2866 *this->matrix_free,
2867 *this->dof_info);
2868 }
2869
2870 const bool accesses_exterior_dofs =
2871 this->dof_access_index ==
2872 internal::MatrixFreeFunctions::DoFInfo::dof_access_cell &&
2873 this->is_interior_face() == false;
2874
2875 // Case 2: contiguous indices which use reduced storage of indices and can
2876 // use vectorized load/store operations -> go to separate function
2877 if (this->cell != numbers::invalid_unsigned_int)
2878 {
2879 AssertIndexRange(
2880 this->cell,
2881 dof_info.index_storage_variants[this->dof_access_index].size());
2882
2883 bool is_contiguous = true;
2884 // check if exterior cells are not contiguous (ECL case)
2885 if (accesses_exterior_dofs)
2886 {
2887 const std::array<unsigned int, n_lanes> &cells = this->get_cell_ids();
2888 const unsigned int n_filled_lanes =
2889 dof_info.n_vectorization_lanes_filled
2890 [internal::MatrixFreeFunctions::DoFInfo::dof_access_cell]
2891 [this->cell];
2892 // we have to check all filled lanes which are active in the mask
2893 for (unsigned int v = 0; v < n_filled_lanes; ++v)
2894 if (mask[v] == true &&
2895 dof_info.index_storage_variants
2896 [internal::MatrixFreeFunctions::DoFInfo::dof_access_cell]
2897 [cells[v] / n_lanes] <
2898 internal::MatrixFreeFunctions::DoFInfo::IndexStorageVariants::
2899 contiguous)
2900 is_contiguous = false;
2901 } // or if cell/face batch is not contiguous
2902 else if (dof_info.index_storage_variants
2903 [is_face ?
2904 this->dof_access_index :
2905 internal::MatrixFreeFunctions::DoFInfo::dof_access_cell]
2906 [this->cell] < internal::MatrixFreeFunctions::DoFInfo::
2907 IndexStorageVariants::contiguous)
2908 {
2909 is_contiguous = false;
2910 }
2911
2912 if (is_contiguous)
2913 {
2914 read_write_operation_contiguous(operation, src, src_sm, mask);
2915 return;
2916 }
2917 }
2918
2919 // Case 3: standard operation with one index per degree of freedom -> go on
2920 // here
2921 std::array<unsigned int, n_lanes> cells = this->get_cell_ids();
2922
2923 const bool masking_is_active = mask.count() < n_lanes;
2924 if (masking_is_active)
2925 for (unsigned int v = 0; v < n_lanes; ++v)
2926 if (mask[v] == false)
2927 cells[v] = numbers::invalid_unsigned_int;
2928
2929 bool has_hn_constraints = false;
2930
2931 if (is_face == false)
2932 {
2933 if (!dof_info.hanging_node_constraint_masks.empty() &&
2934 !dof_info.hanging_node_constraint_masks_comp.empty() &&
2935 dof_info
2936 .hanging_node_constraint_masks_comp[this->active_fe_index]
2937 [this->first_selected_component])
2938 for (unsigned int v = 0; v < n_lanes; ++v)
2939 if (cells[v] != numbers::invalid_unsigned_int &&
2940 dof_info.hanging_node_constraint_masks[cells[v]] !=
2941 internal::MatrixFreeFunctions::
2942 unconstrained_compressed_constraint_kind)
2943 has_hn_constraints = true;
2944 }
2945
2946 std::bool_constant<internal::is_vectorizable<VectorType, Number>::value>
2947 vector_selector;
2948
2949 const bool use_vectorized_path =
2950 !(masking_is_active || has_hn_constraints || accesses_exterior_dofs);
2951
2952 const std::size_t dofs_per_component = this->data->dofs_per_component_on_cell;
2953 std::array<VectorizedArrayType *, n_components> values_dofs;
2954 for (unsigned int c = 0; c < n_components; ++c)
2955 values_dofs[c] = const_cast<VectorizedArrayType *>(this->values_dofs) +
2956 c * dofs_per_component;
2957
2958 if (this->cell != numbers::invalid_unsigned_int &&
2959 dof_info.index_storage_variants
2960 [is_face ? this->dof_access_index :
2961 internal::MatrixFreeFunctions::DoFInfo::dof_access_cell]
2962 [this->cell] == internal::MatrixFreeFunctions::DoFInfo::
2963 IndexStorageVariants::interleaved &&
2964 use_vectorized_path)
2965 {
2966 const unsigned int *dof_indices =
2967 dof_info.dof_indices_interleaved.data() +
2968 dof_info.row_starts[this->cell * this->n_fe_components * n_lanes]
2969 .first +
2970 this->dof_info
2971 ->component_dof_indices_offset[this->active_fe_index]
2972 [this->first_selected_component] *
2973 n_lanes;
2974
2975 std::array<typename VectorType::value_type *, n_components> src_ptrs;
2976 if (n_components == 1 || this->n_fe_components == 1)
2977 for (unsigned int comp = 0; comp < n_components; ++comp)
2978 src_ptrs[comp] =
2979 const_cast<typename VectorType::value_type *>(src[comp]->begin());
2980 else
2981 src_ptrs[0] =
2982 const_cast<typename VectorType::value_type *>(src[0]->begin());
2983
2984 if (n_components == 1 || this->n_fe_components == 1)
2985 for (unsigned int i = 0; i < dofs_per_component;
2986 ++i, dof_indices += n_lanes)
2987 for (unsigned int comp = 0; comp < n_components; ++comp)
2988 operation.process_dof_gather(dof_indices,
2989 *src[comp],
2990 0,
2991 src_ptrs[comp],
2992 values_dofs[comp][i],
2993 vector_selector);
2994 else
2995 for (unsigned int comp = 0; comp < n_components; ++comp)
2996 for (unsigned int i = 0; i < dofs_per_component;
2997 ++i, dof_indices += n_lanes)
2998 operation.process_dof_gather(dof_indices,
2999 *src[0],
3000 0,
3001 src_ptrs[0],
3002 values_dofs[comp][i],
3003 vector_selector);
3004 return;
3005 }
3006
3007 // Allocate pointers, then initialize all of them to nullptrs and
3008 // below overwrite the ones we actually use:
3009 std::array<const unsigned int *, n_lanes> dof_indices;
3010 dof_indices.fill(nullptr);
3011
3012 // Assign the appropriate cell ids for face/cell case and get the pointers
3013 // to the dof indices of the cells on all lanes
3014
3015 bool has_constraints = false;
3016 const unsigned int n_components_read =
3017 this->n_fe_components > 1 ? n_components : 1;
3018
3019 if (is_face)
3020 {
3021 for (unsigned int v = 0; v < n_lanes; ++v)
3022 {
3023 if (cells[v] == numbers::invalid_unsigned_int)
3024 continue;
3025
3026 Assert(cells[v] < dof_info.row_starts.size() - 1, ExcInternalError());
3027 const std::pair<unsigned int, unsigned int> *my_index_start =
3028 &dof_info.row_starts[cells[v] * this->n_fe_components +
3029 this->first_selected_component];
3030
3031 // check whether any of the SIMD lanes has constraints, i.e., the
3032 // constraint indicator which is the second entry of row_starts
3033 // increments on this cell
3034 if (my_index_start[n_components_read].second !=
3035 my_index_start[0].second)
3036 has_constraints = true;
3037
3038 dof_indices[v] =
3039 dof_info.dof_indices.data() + my_index_start[0].first;
3040 }
3041 }
3042 else
3043 {
3044 for (unsigned int v = 0; v < n_lanes; ++v)
3045 {
3046 if (cells[v] == numbers::invalid_unsigned_int)
3047 continue;
3048
3049 const std::pair<unsigned int, unsigned int> *my_index_start =
3050 &dof_info.row_starts[cells[v] * this->n_fe_components +
3051 this->first_selected_component];
3052 if (my_index_start[n_components_read].second !=
3053 my_index_start[0].second)
3054 has_constraints = true;
3055
3056 if (dof_info.hanging_node_constraint_masks.size() > 0 &&
3057 dof_info.hanging_node_constraint_masks_comp.size() > 0 &&
3058 dof_info.hanging_node_constraint_masks[cells[v]] !=
3059 internal::MatrixFreeFunctions::
3060 unconstrained_compressed_constraint_kind &&
3061 dof_info.hanging_node_constraint_masks_comp
3062 [this->active_fe_index][this->first_selected_component])
3063 has_hn_constraints = true;
3064
3065 Assert(my_index_start[n_components_read].first ==
3066 my_index_start[0].first ||
3067 my_index_start[0].first < dof_info.dof_indices.size(),
3068 ExcIndexRange(0,
3069 my_index_start[0].first,
3070 dof_info.dof_indices.size()));
3071 dof_indices[v] =
3072 dof_info.dof_indices.data() + my_index_start[0].first;
3073 }
3074 }
3075
3076 if (std::count_if(cells.begin(), cells.end(), [](const auto i) {
3077 return i != numbers::invalid_unsigned_int;
3078 }) < n_lanes)
3079 for (unsigned int comp = 0; comp < n_components; ++comp)
3080 for (unsigned int i = 0; i < dofs_per_component; ++i)
3081 operation.process_empty(values_dofs[comp][i]);
3082
3083 // Case where we have no constraints throughout the whole cell: Can go
3084 // through the list of DoFs directly
3085 if (!has_constraints && apply_constraints)
3086 {
3087 if (n_components == 1 || this->n_fe_components == 1)
3088 {
3089 for (unsigned int v = 0; v < n_lanes; ++v)
3090 {
3091 if (cells[v] == numbers::invalid_unsigned_int)
3092 continue;
3093
3094 for (unsigned int i = 0; i < dofs_per_component; ++i)
3095 for (unsigned int comp = 0; comp < n_components; ++comp)
3096 operation.process_dof(dof_indices[v][i],
3097 *src[comp],
3098 values_dofs[comp][i][v]);
3099 }
3100 }
3101 else
3102 {
3103 for (unsigned int comp = 0; comp < n_components; ++comp)
3104 for (unsigned int v = 0; v < n_lanes; ++v)
3105 {
3106 if (cells[v] == numbers::invalid_unsigned_int)
3107 continue;
3108
3109 for (unsigned int i = 0; i < dofs_per_component; ++i)
3110 operation.process_dof(
3111 dof_indices[v][comp * dofs_per_component + i],
3112 *src[0],
3113 values_dofs[comp][i][v]);
3114 }
3115 }
3116 return;
3117 }
3118
3119 // In the case where there are some constraints to be resolved, loop over
3120 // all vector components that are filled and then over local dofs. ind_local
3121 // holds local number on cell, index iterates over the elements of
3122 // index_local_to_global and dof_indices points to the global indices stored
3123 // in index_local_to_global
3124
3125 for (unsigned int v = 0; v < n_lanes; ++v)
3126 {
3127 if (cells[v] == numbers::invalid_unsigned_int)
3128 continue;
3129
3130 const unsigned int cell_index = cells[v];
3131 const unsigned int cell_dof_index =
3132 cell_index * this->n_fe_components + this->first_selected_component;
3133 const unsigned int n_components_read =
3134 this->n_fe_components > 1 ? n_components : 1;
3135 unsigned int index_indicators =
3136 dof_info.row_starts[cell_dof_index].second;
3137 unsigned int next_index_indicators =
3138 dof_info.row_starts[cell_dof_index + 1].second;
3139
3140 // For read_dof_values_plain, redirect the dof_indices field to the
3141 // unconstrained indices
3142 if (apply_constraints == false &&
3143 (dof_info.row_starts[cell_dof_index].second !=
3144 dof_info.row_starts[cell_dof_index + n_components_read].second ||
3145 ((dof_info.hanging_node_constraint_masks.size() > 0 &&
3146 dof_info.hanging_node_constraint_masks_comp.size() > 0 &&
3147 dof_info.hanging_node_constraint_masks[cell_index] !=
3148 internal::MatrixFreeFunctions::
3149 unconstrained_compressed_constraint_kind) &&
3150 dof_info.hanging_node_constraint_masks_comp
3151 [this->active_fe_index][this->first_selected_component])))
3152 {
3153 Assert(dof_info.row_starts_plain_indices[cell_index] !=
3154 numbers::invalid_unsigned_int,
3155 ExcNotInitialized());
3156 dof_indices[v] =
3157 dof_info.plain_dof_indices.data() +
3158 this->dof_info
3159 ->component_dof_indices_offset[this->active_fe_index]
3160 [this->first_selected_component] +
3161 dof_info.row_starts_plain_indices[cell_index];
3162 next_index_indicators = index_indicators;
3163 }
3164
3165 if (n_components == 1 || this->n_fe_components == 1)
3166 {
3167 unsigned int ind_local = 0;
3168 for (; index_indicators != next_index_indicators; ++index_indicators)
3169 {
3170 const std::pair<unsigned short, unsigned short> indicator =
3171 dof_info.constraint_indicator[index_indicators];
3172 // run through values up to next constraint
3173 for (unsigned int j = 0; j < indicator.first; ++j)
3174 for (unsigned int comp = 0; comp < n_components; ++comp)
3175 operation.process_dof(dof_indices[v][j],
3176 *src[comp],
3177 values_dofs[comp][ind_local + j][v]);
3178
3179 ind_local += indicator.first;
3180 dof_indices[v] += indicator.first;
3181
3182 // constrained case: build the local value as a linear
3183 // combination of the global value according to constraints
3184 Number value[n_components];
3185 for (unsigned int comp = 0; comp < n_components; ++comp)
3186 operation.pre_constraints(values_dofs[comp][ind_local][v],
3187 value[comp]);
3188
3189 const Number *data_val =
3190 this->matrix_free->constraint_pool_begin(indicator.second);
3191 const Number *end_pool =
3192 this->matrix_free->constraint_pool_end(indicator.second);
3193 for (; data_val != end_pool; ++data_val, ++dof_indices[v])
3194 for (unsigned int comp = 0; comp < n_components; ++comp)
3195 operation.process_constraint(*dof_indices[v],
3196 *data_val,
3197 *src[comp],
3198 value[comp]);
3199
3200 for (unsigned int comp = 0; comp < n_components; ++comp)
3201 operation.post_constraints(value[comp],
3202 values_dofs[comp][ind_local][v]);
3203 ++ind_local;
3204 }
3205
3206 AssertIndexRange(ind_local, dofs_per_component + 1);
3207
3208 for (; ind_local < dofs_per_component; ++dof_indices[v], ++ind_local)
3209 for (unsigned int comp = 0; comp < n_components; ++comp)
3210 operation.process_dof(*dof_indices[v],
3211 *src[comp],
3212 values_dofs[comp][ind_local][v]);
3213 }
3214 else
3215 {
3216 // case with vector-valued finite elements where all components are
3217 // included in one single vector. Assumption: first come all entries
3218 // to the first component, then all entries to the second one, and
3219 // so on. This is ensured by the way MatrixFree reads out the
3220 // indices.
3221 for (unsigned int comp = 0; comp < n_components; ++comp)
3222 {
3223 unsigned int ind_local = 0;
3224
3225 // check whether there is any constraint on the current cell
3226 for (; index_indicators != next_index_indicators;
3227 ++index_indicators)
3228 {
3229 const std::pair<unsigned short, unsigned short> indicator =
3230 dof_info.constraint_indicator[index_indicators];
3231
3232 // run through values up to next constraint
3233 for (unsigned int j = 0; j < indicator.first; ++j)
3234 operation.process_dof(dof_indices[v][j],
3235 *src[0],
3236 values_dofs[comp][ind_local + j][v]);
3237 ind_local += indicator.first;
3238 dof_indices[v] += indicator.first;
3239
3240 // constrained case: build the local value as a linear
3241 // combination of the global value according to constraints
3242 Number value;
3243 operation.pre_constraints(values_dofs[comp][ind_local][v],
3244 value);
3245
3246 const Number *data_val =
3247 this->matrix_free->constraint_pool_begin(indicator.second);
3248 const Number *end_pool =
3249 this->matrix_free->constraint_pool_end(indicator.second);
3250
3251 for (; data_val != end_pool; ++data_val, ++dof_indices[v])
3252 operation.process_constraint(*dof_indices[v],
3253 *data_val,
3254 *src[0],
3255 value);
3256
3257 operation.post_constraints(value,
3258 values_dofs[comp][ind_local][v]);
3259 ++ind_local;
3260 }
3261
3262 AssertIndexRange(ind_local, dofs_per_component + 1);
3263
3264 // get the dof values past the last constraint
3265 for (; ind_local < dofs_per_component;
3266 ++dof_indices[v], ++ind_local)
3267 {
3268 AssertIndexRange(*dof_indices[v], src[0]->size());
3269 operation.process_dof(*dof_indices[v],
3270 *src[0],
3271 values_dofs[comp][ind_local][v]);
3272 }
3273
3274 if (apply_constraints == true && comp + 1 < n_components)
3275 next_index_indicators =
3276 dof_info.row_starts[cell_dof_index + comp + 2].second;
3277 }
3278 }
3279 }
3280 }
3281
3282
3283
3284 template <int dim,
3285 int n_components_,
3286 typename Number,
3287 bool is_face,
3288 typename VectorizedArrayType>
3289 template <typename VectorType, typename VectorOperation>
3290 inline void
3291 FEEvaluationBase<dim, n_components_, Number, is_face, VectorizedArrayType>::
3292 read_write_operation_global(
3293 const VectorOperation &operation,
3294 const std::array<VectorType *, n_components_> &src) const
3295 {
3296 Assert(!local_dof_indices.empty(), ExcNotInitialized());
3297
3298 const std::size_t dofs_per_component = this->data->dofs_per_component_on_cell;
3299 unsigned int index = this->first_selected_component * dofs_per_component;
3300 for (unsigned int comp = 0; comp < n_components; ++comp)
3301 {
3302 for (unsigned int i = 0; i < dofs_per_component; ++i, ++index)
3303 {
3304 operation.process_empty(
3305 this->values_dofs[comp * dofs_per_component + i]);
3306 operation.process_dof_global(
3307 local_dof_indices[this->data->lexicographic_numbering[index]],
3308 *src[0],
3309 this->values_dofs[comp * dofs_per_component + i][0]);
3310 }
3311 }
3312 }
3313
3314
3315
3316 template <int dim,
3317 int n_components_,
3318 typename Number,
3319 bool is_face,
3320 typename VectorizedArrayType>
3321 template <typename VectorType, typename VectorOperation>
3322 inline void
3323 FEEvaluationBase<dim, n_components_, Number, is_face, VectorizedArrayType>::
3324 read_write_operation_contiguous(
3325 const VectorOperation &operation,
3326 const std::array<VectorType *, n_components_> &src,
3327 const std::array<
3328 const std::vector<ArrayView<const typename VectorType::value_type>> *,
3329 n_components_> &vectors_sm,
3330 const std::bitset<n_lanes> &mask) const
3331 {
3332 // This functions processes the functions read_dof_values,
3333 // distribute_local_to_global, and set_dof_values with the same code for
3334 // contiguous cell indices (DG case). The distinction between these three
3335 // cases is made by the input VectorOperation that either reads values from
3336 // a vector and puts the data into the local data field or write local data
3337 // into the vector. Certain operations are no-ops for the given use case.
3338
3339 std::bool_constant<internal::is_vectorizable<VectorType, Number>::value>
3340 vector_selector;
3341 const internal::MatrixFreeFunctions::DoFInfo::DoFAccessIndex ind =
3342 is_face ? this->dof_access_index :
3343 internal::MatrixFreeFunctions::DoFInfo::dof_access_cell;
3344 const unsigned int n_active_lanes = mask.count();
3345
3346 const internal::MatrixFreeFunctions::DoFInfo &dof_info = *this->dof_info;
3347 const std::vector<unsigned int> &dof_indices_cont =
3348 dof_info.dof_indices_contiguous[ind];
3349
3350 const std::size_t dofs_per_component = this->data->dofs_per_component_on_cell;
3351 std::array<VectorizedArrayType *, n_components> values_dofs{{nullptr}};
3352 for (unsigned int c = 0; c < n_components; ++c)
3353 values_dofs[c] = const_cast<VectorizedArrayType *>(this->values_dofs) +
3354 c * dofs_per_component;
3355
3356 Assert(this->cell != numbers::invalid_unsigned_int, ExcNotImplemented());
3357
3358 const bool accesses_exterior_dofs =
3359 this->dof_access_index ==
3360 internal::MatrixFreeFunctions::DoFInfo::dof_access_cell &&
3361 this->is_interior_face() == false;
3362
3363 // Simple case: We have contiguous storage, so we can simply copy out the
3364 // data
3365 if (dof_info.index_storage_variants[ind][this->cell] ==
3366 internal::MatrixFreeFunctions::DoFInfo::IndexStorageVariants::
3367 interleaved_contiguous &&
3368 n_active_lanes == n_lanes && !accesses_exterior_dofs)
3369 {
3370 const unsigned int dof_index =
3371 dof_indices_cont[this->cell * n_lanes] +
3372 this->dof_info
3373 ->component_dof_indices_offset[this->active_fe_index]
3374 [this->first_selected_component] *
3375 n_lanes;
3376 if (n_components == 1 || this->n_fe_components == 1)
3377 for (unsigned int comp = 0; comp < n_components; ++comp)
3378 operation.process_dofs_vectorized(dofs_per_component,
3379 dof_index,
3380 *src[comp],
3381 values_dofs[comp],
3382 vector_selector);
3383 else
3384 operation.process_dofs_vectorized(dofs_per_component * n_components,
3385 dof_index,
3386 *src[0],
3387 values_dofs[0],
3388 vector_selector);
3389 return;
3390 }
3391
3392 const std::array<unsigned int, n_lanes> &cells = this->get_cell_or_face_ids();
3393
3394 // More general case: Must go through the components one by one and apply
3395 // some transformations
3396 const unsigned int n_filled_lanes =
3397 dof_info.n_vectorization_lanes_filled[ind][this->cell];
3398
3399 const bool use_vectorized_path = n_filled_lanes == n_lanes &&
3400 n_active_lanes == n_lanes &&
3401 !accesses_exterior_dofs;
3402
3403 if (vectors_sm[0] != nullptr)
3404 {
3405 const auto compute_vector_ptrs = [&](const unsigned int comp) {
3406 std::array<typename VectorType::value_type *, n_lanes> vector_ptrs{
3407 {nullptr}};
3408
3409 const auto upper_bound =
3410 std::min<unsigned int>(n_filled_lanes, n_lanes);
3411 for (unsigned int v = 0; v < upper_bound; ++v)
3412 {
3413 if (mask[v] == false)
3414 {
3415 vector_ptrs[v] = nullptr;
3416 continue;
3417 }
3418
3419 Assert(cells[v] != numbers::invalid_unsigned_int,
3420 ExcNotImplemented());
3421 Assert(ind < dof_info.dof_indices_contiguous_sm.size(),
3422 ExcIndexRange(ind,
3423 0,
3424 dof_info.dof_indices_contiguous_sm.size()));
3425 Assert(
3426 cells[v] < dof_info.dof_indices_contiguous_sm[ind].size(),
3427 ExcIndexRange(cells[v],
3428 0,
3429 dof_info.dof_indices_contiguous_sm[ind].size()));
3430
3431 const auto &temp =
3432 dof_info.dof_indices_contiguous_sm[ind][cells[v]];
3433
3434 if (temp.first != numbers::invalid_unsigned_int)
3435 vector_ptrs[v] = const_cast<typename VectorType::value_type *>(
3436 vectors_sm[comp]->operator[](temp.first).data() + temp.second +
3437 dof_info.component_dof_indices_offset
3438 [this->active_fe_index][this->first_selected_component]);
3439 else
3440 vector_ptrs[v] = nullptr;
3441 }
3442 for (unsigned int v = n_filled_lanes; v < n_lanes; ++v)
3443 vector_ptrs[v] = nullptr;
3444
3445 return vector_ptrs;
3446 };
3447
3448 if (use_vectorized_path)
3449 {
3450 if (n_components == 1 || this->n_fe_components == 1)
3451 {
3452 for (unsigned int comp = 0; comp < n_components; ++comp)
3453 {
3454 auto vector_ptrs = compute_vector_ptrs(comp);
3455 operation.process_dofs_vectorized_transpose(
3456 dofs_per_component,
3457 vector_ptrs,
3458 values_dofs[comp],
3459 vector_selector);
3460 }
3461 }
3462 else
3463 {
3464 auto vector_ptrs = compute_vector_ptrs(0);
3465 operation.process_dofs_vectorized_transpose(dofs_per_component *
3466 n_components,
3467 vector_ptrs,
3468 &values_dofs[0][0],
3469 vector_selector);
3470 }
3471 }
3472 else
3473 for (unsigned int comp = 0; comp < n_components; ++comp)
3474 {
3475 auto vector_ptrs = compute_vector_ptrs(
3476 (n_components == 1 || this->n_fe_components == 1) ? comp : 0);
3477
3478 for (unsigned int i = 0; i < dofs_per_component; ++i)
3479 operation.process_empty(values_dofs[comp][i]);
3480
3481 if (n_components == 1 || this->n_fe_components == 1)
3482 {
3483 for (unsigned int v = 0; v < n_filled_lanes; ++v)
3484 if (mask[v] == true)
3485 for (unsigned int i = 0; i < dofs_per_component; ++i)
3486 operation.process_dof(vector_ptrs[v][i],
3487 values_dofs[comp][i][v]);
3488 }
3489 else
3490 {
3491 for (unsigned int v = 0; v < n_filled_lanes; ++v)
3492 if (mask[v] == true)
3493 for (unsigned int i = 0; i < dofs_per_component; ++i)
3494 operation.process_dof(
3495 vector_ptrs[v][i + comp * dofs_per_component],
3496 values_dofs[comp][i][v]);
3497 }
3498 }
3499 return;
3500 }
3501
3502 std::array<unsigned int, n_lanes> dof_indices{
3503 {numbers::invalid_unsigned_int}};
3504 Assert(n_filled_lanes <= n_lanes, ExcInternalError());
3505 for (unsigned int v = 0; v < n_filled_lanes; ++v)
3506 {
3507 Assert(mask[v] == false || cells[v] != numbers::invalid_unsigned_int,
3508 ExcNotImplemented());
3509 if (mask[v] == true)
3510 dof_indices[v] =
3511 dof_indices_cont[cells[v]] +
3512 this->dof_info
3513 ->component_dof_indices_offset[this->active_fe_index]
3514 [this->first_selected_component] *
3515 dof_info.dof_indices_interleave_strides[ind][cells[v]];
3516 }
3517
3518 // In the case with contiguous cell indices, we know that there are no
3519 // constraints and that the indices within each element are contiguous
3520 if (use_vectorized_path)
3521 {
3522 if (dof_info.index_storage_variants[ind][this->cell] ==
3523 internal::MatrixFreeFunctions::DoFInfo::IndexStorageVariants::
3524 contiguous)
3525 {
3526 if (n_components == 1 || this->n_fe_components == 1)
3527 for (unsigned int comp = 0; comp < n_components; ++comp)
3528 operation.process_dofs_vectorized_transpose(dofs_per_component,
3529 dof_indices.data(),
3530 *src[comp],
3531 values_dofs[comp],
3532 vector_selector);
3533 else
3534 operation.process_dofs_vectorized_transpose(dofs_per_component *
3535 n_components,
3536 dof_indices.data(),
3537 *src[0],
3538 &values_dofs[0][0],
3539 vector_selector);
3540 }
3541 else if (dof_info.index_storage_variants[ind][this->cell] ==
3542 internal::MatrixFreeFunctions::DoFInfo::IndexStorageVariants::
3543 interleaved_contiguous_strided)
3544 {
3545 std::array<typename VectorType::value_type *, n_components> src_ptrs{
3546 {nullptr}};
3547 if (n_components == 1 || this->n_fe_components == 1)
3548 for (unsigned int comp = 0; comp < n_components; ++comp)
3549 src_ptrs[comp] = const_cast<typename VectorType::value_type *>(
3550 src[comp]->begin());
3551 else
3552 src_ptrs[0] =
3553 const_cast<typename VectorType::value_type *>(src[0]->begin());
3554
3555 if (n_components == 1 || this->n_fe_components == 1)
3556 for (unsigned int i = 0; i < dofs_per_component; ++i)
3557 {
3558 for (unsigned int comp = 0; comp < n_components; ++comp)
3559 operation.process_dof_gather(dof_indices.data(),
3560 *src[comp],
3561 i * n_lanes,
3562 src_ptrs[comp] + i * n_lanes,
3563 values_dofs[comp][i],
3564 vector_selector);
3565 }
3566 else
3567 for (unsigned int comp = 0; comp < n_components; ++comp)
3568 for (unsigned int i = 0; i < dofs_per_component; ++i)
3569 {
3570 operation.process_dof_gather(
3571 dof_indices.data(),
3572 *src[0],
3573 (comp * dofs_per_component + i) * n_lanes,
3574 src_ptrs[0] + (comp * dofs_per_component + i) * n_lanes,
3575 values_dofs[comp][i],
3576 vector_selector);
3577 }
3578 }
3579 else
3580 {
3581 Assert(dof_info.index_storage_variants[ind][this->cell] ==
3582 internal::MatrixFreeFunctions::DoFInfo::
3583 IndexStorageVariants::interleaved_contiguous_mixed_strides,
3584 ExcNotImplemented());
3585 std::array<typename VectorType::value_type *, n_components> src_ptrs{
3586 {nullptr}};
3587 if (n_components == 1 || this->n_fe_components == 1)
3588 for (unsigned int comp = 0; comp < n_components; ++comp)
3589 src_ptrs[comp] = const_cast<typename VectorType::value_type *>(
3590 src[comp]->begin());
3591 else
3592 src_ptrs[0] =
3593 const_cast<typename VectorType::value_type *>(src[0]->begin());
3594
3595 const unsigned int *offsets =
3596 &dof_info.dof_indices_interleave_strides[ind][n_lanes * this->cell];
3597 if (n_components == 1 || this->n_fe_components == 1)
3598 for (unsigned int i = 0; i < dofs_per_component; ++i)
3599 {
3600 for (unsigned int comp = 0; comp < n_components; ++comp)
3601 operation.process_dof_gather(dof_indices.data(),
3602 *src[comp],
3603 0,
3604 src_ptrs[comp],
3605 values_dofs[comp][i],
3606 vector_selector);
3607 DEAL_II_OPENMP_SIMD_PRAGMA
3608 for (unsigned int v = 0; v < n_lanes; ++v)
3609 dof_indices[v] += offsets[v];
3610 }
3611 else
3612 for (unsigned int comp = 0; comp < n_components; ++comp)
3613 for (unsigned int i = 0; i < dofs_per_component; ++i)
3614 {
3615 operation.process_dof_gather(dof_indices.data(),
3616 *src[0],
3617 0,
3618 src_ptrs[0],
3619 values_dofs[comp][i],
3620 vector_selector);
3621 DEAL_II_OPENMP_SIMD_PRAGMA
3622 for (unsigned int v = 0; v < n_lanes; ++v)
3623 dof_indices[v] += offsets[v];
3624 }
3625 }
3626 }
3627 else
3628 for (unsigned int comp = 0; comp < n_components; ++comp)
3629 {
3630 for (unsigned int i = 0; i < dofs_per_component; ++i)
3631 operation.process_empty(values_dofs[comp][i]);
3632 if (accesses_exterior_dofs)
3633 {
3634 for (unsigned int v = 0; v < n_filled_lanes; ++v)
3635 if (mask[v] == true)
3636 {
3637 if (dof_info.index_storage_variants
3638 [ind][cells[v] / VectorizedArrayType::size()] ==
3639 internal::MatrixFreeFunctions::DoFInfo::
3640 IndexStorageVariants::contiguous)
3641 {
3642 if (n_components == 1 || this->n_fe_components == 1)
3643 {
3644 for (unsigned int i = 0; i < dofs_per_component; ++i)
3645 operation.process_dof(dof_indices[v] + i,
3646 *src[comp],
3647 values_dofs[comp][i][v]);
3648 }
3649 else
3650 {
3651 for (unsigned int i = 0; i < dofs_per_component; ++i)
3652 operation.process_dof(dof_indices[v] + i +
3653 comp * dofs_per_component,
3654 *src[0],
3655 values_dofs[comp][i][v]);
3656 }
3657 }
3658 else
3659 {
3660 const unsigned int offset =
3661 dof_info.dof_indices_interleave_strides[ind][cells[v]];
3662 AssertIndexRange(offset, VectorizedArrayType::size() + 1);
3663 if (n_components == 1 || this->n_fe_components == 1)
3664 {
3665 for (unsigned int i = 0; i < dofs_per_component; ++i)
3666 operation.process_dof(dof_indices[v] + i * offset,
3667 *src[comp],
3668 values_dofs[comp][i][v]);
3669 }
3670 else
3671 {
3672 for (unsigned int i = 0; i < dofs_per_component; ++i)
3673 operation.process_dof(
3674 dof_indices[v] +
3675 (i + comp * dofs_per_component) * offset,
3676 *src[0],
3677 values_dofs[comp][i][v]);
3678 }
3679 }
3680 }
3681 }
3682 else
3683 {
3684 if (dof_info.index_storage_variants[ind][this->cell] ==
3685 internal::MatrixFreeFunctions::DoFInfo::IndexStorageVariants::
3686 contiguous)
3687 {
3688 if (n_components == 1 || this->n_fe_components == 1)
3689 {
3690 for (unsigned int v = 0; v < n_filled_lanes; ++v)
3691 if (mask[v] == true)
3692 for (unsigned int i = 0; i < dofs_per_component; ++i)
3693 operation.process_dof(dof_indices[v] + i,
3694 *src[comp],
3695 values_dofs[comp][i][v]);
3696 }
3697 else
3698 {
3699 for (unsigned int v = 0; v < n_filled_lanes; ++v)
3700 if (mask[v] == true)
3701 for (unsigned int i = 0; i < dofs_per_component; ++i)
3702 operation.process_dof(dof_indices[v] + i +
3703 comp * dofs_per_component,
3704 *src[0],
3705 values_dofs[comp][i][v]);
3706 }
3707 }
3708 else
3709 {
3710 const unsigned int *offsets =
3711 &dof_info.dof_indices_interleave_strides
3712 [ind][VectorizedArrayType::size() * this->cell];
3713 for (unsigned int v = 0; v < n_filled_lanes; ++v)
3714 AssertIndexRange(offsets[v], VectorizedArrayType::size() + 1);
3715 if (n_components == 1 || this->n_fe_components == 1)
3716 for (unsigned int v = 0; v < n_filled_lanes; ++v)
3717 {
3718 if (mask[v] == true)
3719 for (unsigned int i = 0; i < dofs_per_component; ++i)
3720 operation.process_dof(dof_indices[v] + i * offsets[v],
3721 *src[comp],
3722 values_dofs[comp][i][v]);
3723 }
3724 else
3725 {
3726 for (unsigned int v = 0; v < n_filled_lanes; ++v)
3727 if (mask[v] == true)
3728 for (unsigned int i = 0; i < dofs_per_component; ++i)
3729 operation.process_dof(
3730 dof_indices[v] +
3731 (i + comp * dofs_per_component) * offsets[v],
3732 *src[0],
3733 values_dofs[comp][i][v]);
3734 }
3735 }
3736 }
3737 }
3738 }
3739
3740 namespace internal
3741 {
3742 template <
3743 typename Number,
3744 typename VectorType,
3745 std::enable_if_t<!IsBlockVector<VectorType>::value, VectorType> * = nullptr>
3746 decltype(std::declval<VectorType>().begin())
3747 get_beginning(VectorType &vec)
3748 {
3749 return vec.begin();
3750 }
3751
3752 template <
3753 typename Number,
3754 typename VectorType,
3755 std::enable_if_t<IsBlockVector<VectorType>::value, VectorType> * = nullptr>
3756 typename VectorType::value_type *
3757 get_beginning(VectorType &)
3758 {
3759 return nullptr;
3760 }
3761
3762 template <typename VectorType,
3763 std::enable_if_t<has_shared_vector_data<VectorType>, VectorType> * =
3764 nullptr>
3765 const std::vector<ArrayView<const typename VectorType::value_type>> *
3766 get_shared_vector_data(VectorType *vec,
3767 const bool is_valid_mode_for_sm,
3768 const unsigned int active_fe_index,
3769 const internal::MatrixFreeFunctions::DoFInfo *dof_info)
3770 {
3771 // note: no hp is supported
3772 if (is_valid_mode_for_sm &&
3773 dof_info->dof_indices_contiguous_sm[0 /*any index (<3) should work*/]
3774 .size() > 0 &&
3775 active_fe_index == 0)
3776 return &vec->shared_vector_data();
3777 else
3778 return nullptr;
3779 }
3780
3781 template <typename VectorType,
3782 std::enable_if_t<!has_shared_vector_data<VectorType>, VectorType>
3783 * = nullptr>
3784 const std::vector<ArrayView<const typename VectorType::value_type>> *
3785 get_shared_vector_data(VectorType *,
3786 const bool,
3787 const unsigned int,
3788 const internal::MatrixFreeFunctions::DoFInfo *)
3789 {
3790 return nullptr;
3791 }
3792
3793 template <int n_components, typename VectorType>
3794 std::pair<
3795 std::array<typename internal::BlockVectorSelector<
3796 VectorType,
3797 IsBlockVector<VectorType>::value>::BaseVectorType *,
3798 n_components>,
3799 std::array<
3800 const std::vector<ArrayView<const typename internal::BlockVectorSelector<
3801 VectorType,
3802 IsBlockVector<VectorType>::value>::BaseVectorType::value_type>> *,
3803 n_components>>
3804 get_vector_data(VectorType &src,
3805 const unsigned int first_index,
3806 const bool is_valid_mode_for_sm,
3807 const unsigned int active_fe_index,
3808 const internal::MatrixFreeFunctions::DoFInfo *dof_info)
3809 {
3810 // select between block vectors and non-block vectors. Note that the number
3811 // of components is checked in the internal data
3812 std::pair<
3813 std::array<typename internal::BlockVectorSelector<
3814 VectorType,
3815 IsBlockVector<VectorType>::value>::BaseVectorType *,
3816 n_components>,
3817 std::array<
3818 const std::vector<
3819 ArrayView<const typename internal::BlockVectorSelector<
3820 VectorType,
3821 IsBlockVector<VectorType>::value>::BaseVectorType::value_type>> *,
3822 n_components>>
3823 src_data;
3824
3825 for (unsigned int d = 0; d < n_components; ++d)
3826 src_data.first[d] = internal::BlockVectorSelector<
3827 VectorType,
3828 IsBlockVector<VectorType>::value>::get_vector_component(src,
3829 d +
3830 first_index);
3831
3832 for (unsigned int d = 0; d < n_components; ++d)
3833 src_data.second[d] = get_shared_vector_data(
3834 const_cast<typename internal::BlockVectorSelector<
3835 std::remove_const_t<VectorType>,
3836 IsBlockVector<std::remove_const_t<VectorType>>::value>::BaseVectorType
3837 *>(src_data.first[d]),
3838 is_valid_mode_for_sm,
3839 active_fe_index,
3840 dof_info);
3841
3842 return src_data;
3843 }
3844 } // namespace internal
3845
3846
3847
3848 template <int dim,
3849 int n_components_,
3850 typename Number,
3851 bool is_face,
3852 typename VectorizedArrayType>
3853 inline void
3854 FEEvaluationBase<dim, n_components_, Number, is_face, VectorizedArrayType>::
3855 apply_hanging_node_constraints(const bool transpose) const
3856 {
3857 if (this->dof_info == nullptr ||
3858 this->dof_info->hanging_node_constraint_masks.empty() ||
3859 this->dof_info->hanging_node_constraint_masks_comp.empty() ||
3860 this->dof_info->hanging_node_constraint_masks_comp
3861 [this->active_fe_index][this->first_selected_component] == false)
3862 return; // nothing to do with faces
3863
3864 std::array<internal::MatrixFreeFunctions::compressed_constraint_kind, n_lanes>
3865 constraint_mask{{internal::MatrixFreeFunctions::
3866 unconstrained_compressed_constraint_kind}};
3867
3868 bool hn_available = false;
3869
3870 const std::array<unsigned int, n_lanes> &cells = this->get_cell_ids();
3871
3872 for (unsigned int v = 0; v < n_lanes; ++v)
3873 {
3874 if (cells[v] == numbers::invalid_unsigned_int)
3875 {
3876 constraint_mask[v] = internal::MatrixFreeFunctions::
3877 unconstrained_compressed_constraint_kind;
3878 continue;
3879 }
3880
3881 const unsigned int cell_index = cells[v];
3882 const auto mask =
3883 this->dof_info->hanging_node_constraint_masks[cell_index];
3884 constraint_mask[v] = mask;
3885
3886 hn_available |= (mask != internal::MatrixFreeFunctions::
3887 unconstrained_compressed_constraint_kind);
3888 }
3889
3890 if (hn_available == false)
3891 return; // no hanging node on cell batch -> nothing to do
3892
3893 internal::FEEvaluationHangingNodesFactory<dim, Number, VectorizedArrayType>::
3894 apply(n_components,
3895 this->data->data.front().fe_degree,
3896 this->get_shape_info(),
3897 transpose,
3898 constraint_mask,
3899 this->values_dofs);
3900 }
3901
3902
3903
3904 template <int dim,
3905 int n_components_,
3906 typename Number,
3907 bool is_face,
3908 typename VectorizedArrayType>
3909 template <typename VectorType>
3910 inline void
3911 FEEvaluationBase<dim, n_components_, Number, is_face, VectorizedArrayType>::
3912 read_dof_values(const VectorType &src,
3913 const unsigned int first_index,
3914 const std::bitset<n_lanes> &mask)
3915 {
3916 const auto src_data = internal::get_vector_data<n_components_>(
3917 src,
3918 first_index,
3919 this->dof_info != nullptr &&
3920 this->dof_access_index ==
3921 internal::MatrixFreeFunctions::DoFInfo::dof_access_cell,
3922 this->active_fe_index,
3923 this->dof_info);
3924
3925 internal::VectorReader<Number, VectorizedArrayType> reader;
3926 read_write_operation(reader, src_data.first, src_data.second, mask, true);
3927
3928 apply_hanging_node_constraints(false);
3929
3930 if constexpr (running_in_debug_mode())
3931 {
3932 this->dof_values_initialized = true;
3933 }
3934 }
3935
3936
3937
3938 template <int dim,
3939 int n_components_,
3940 typename Number,
3941 bool is_face,
3942 typename VectorizedArrayType>
3943 template <typename VectorType>
3944 inline void
3945 FEEvaluationBase<dim, n_components_, Number, is_face, VectorizedArrayType>::
3946 read_dof_values_plain(const VectorType &src,
3947 const unsigned int first_index,
3948 const std::bitset<n_lanes> &mask)
3949 {
3950 const auto src_data = internal::get_vector_data<n_components_>(
3951 src,
3952 first_index,
3953 this->dof_access_index ==
3954 internal::MatrixFreeFunctions::DoFInfo::dof_access_cell,
3955 this->active_fe_index,
3956 this->dof_info);
3957
3958 internal::VectorReader<Number, VectorizedArrayType> reader;
3959 read_write_operation(reader, src_data.first, src_data.second, mask, false);
3960
3961 if constexpr (running_in_debug_mode())
3962 {
3963 this->dof_values_initialized = true;
3964 }
3965 }
3966
3967
3968
3969 template <int dim,
3970 int n_components_,
3971 typename Number,
3972 bool is_face,
3973 typename VectorizedArrayType>
3974 template <typename VectorType>
3975 inline void
3976 FEEvaluationBase<dim, n_components_, Number, is_face, VectorizedArrayType>::
3977 distribute_local_to_global(VectorType &dst,
3978 const unsigned int first_index,
3979 const std::bitset<n_lanes> &mask) const
3980 {
3981 if constexpr (running_in_debug_mode())
3982 {
3983 Assert(this->dof_values_initialized == true,
3984 internal::ExcAccessToUninitializedField());
3985 }
3986
3987 apply_hanging_node_constraints(true);
3988
3989 const auto dst_data = internal::get_vector_data<n_components_>(
3990 dst,
3991 first_index,
3992 this->dof_access_index ==
3993 internal::MatrixFreeFunctions::DoFInfo::dof_access_cell,
3994 this->active_fe_index,
3995 this->dof_info);
3996
3997 internal::VectorDistributorLocalToGlobal<Number, VectorizedArrayType>
3998 distributor;
3999 read_write_operation(distributor, dst_data.first, dst_data.second, mask);
4000 }
4001
4002
4003
4004 template <int dim,
4005 int n_components_,
4006 typename Number,
4007 bool is_face,
4008 typename VectorizedArrayType>
4009 template <typename VectorType>
4010 inline void
4011 FEEvaluationBase<dim, n_components_, Number, is_face, VectorizedArrayType>::
4012 set_dof_values(VectorType &dst,
4013 const unsigned int first_index,
4014 const std::bitset<n_lanes> &mask) const
4015 {
4016 if constexpr (running_in_debug_mode())
4017 {
4018 Assert(this->dof_values_initialized == true,
4019 internal::ExcAccessToUninitializedField());
4020 }
4021
4022 const auto dst_data = internal::get_vector_data<n_components_>(
4023 dst,
4024 first_index,
4025 this->dof_access_index ==
4026 internal::MatrixFreeFunctions::DoFInfo::dof_access_cell,
4027 this->active_fe_index,
4028 this->dof_info);
4029
4030 internal::VectorSetter<Number, VectorizedArrayType> setter;
4031 read_write_operation(setter, dst_data.first, dst_data.second, mask);
4032 }
4033
4034
4035
4036 template <int dim,
4037 int n_components_,
4038 typename Number,
4039 bool is_face,
4040 typename VectorizedArrayType>
4041 template <typename VectorType>
4042 inline void
4043 FEEvaluationBase<dim, n_components_, Number, is_face, VectorizedArrayType>::
4044 set_dof_values_plain(VectorType &dst,
4045 const unsigned int first_index,
4046 const std::bitset<n_lanes> &mask) const
4047 {
4048 if constexpr (running_in_debug_mode())
4049 {
4050 Assert(this->dof_values_initialized == true,
4051 internal::ExcAccessToUninitializedField());
4052 }
4053
4054 const auto dst_data = internal::get_vector_data<n_components_>(
4055 dst,
4056 first_index,
4057 this->dof_access_index ==
4058 internal::MatrixFreeFunctions::DoFInfo::dof_access_cell,
4059 this->active_fe_index,
4060 this->dof_info);
4061
4062 internal::VectorSetter<Number, VectorizedArrayType> setter;
4063 read_write_operation(setter, dst_data.first, dst_data.second, mask, false);
4064 }
4065
4066
4067
4068 /*------------------------------ access to data fields ----------------------*/
4069
4070
4071
4072 template <int dim,
4073 int n_components_,
4074 typename Number,
4075 bool is_face,
4076 typename VectorizedArrayType>
4077 inline DEAL_II_ALWAYS_INLINE
4078 typename FEEvaluationBase<dim,
4079 n_components_,
4080 Number,
4081 is_face,
4082 VectorizedArrayType>::value_type
4083 FEEvaluationBase<dim, n_components_, Number, is_face, VectorizedArrayType>::
4084 get_dof_value(const unsigned int dof) const
4085 {
4086 AssertIndexRange(dof, this->data->dofs_per_component_on_cell);
4087 if constexpr (n_components == 1)
4088 return this->values_dofs[dof];
4089 else
4090 {
4091 const std::size_t dofs = this->data->dofs_per_component_on_cell;
4092 Tensor<1, n_components_, VectorizedArrayType> return_value;
4093 for (unsigned int comp = 0; comp < n_components; ++comp)
4094 return_value[comp] = this->values_dofs[comp * dofs + dof];
4095 return return_value;
4096 }
4097 }
4098
4099
4100
4101 template <int dim,
4102 int n_components_,
4103 typename Number,
4104 bool is_face,
4105 typename VectorizedArrayType>
4106 inline DEAL_II_ALWAYS_INLINE
4107 typename FEEvaluationBase<dim,
4108 n_components_,
4109 Number,
4110 is_face,
4111 VectorizedArrayType>::value_type
4112 FEEvaluationBase<dim, n_components_, Number, is_face, VectorizedArrayType>::
4113 get_value(const unsigned int q_point) const
4114 {
4115 if constexpr (running_in_debug_mode())
4116 {
4117 Assert(this->values_quad_initialized == true,
4118 internal::ExcAccessToUninitializedField());
4119 }
4120
4121 AssertIndexRange(q_point, this->n_quadrature_points);
4122 if constexpr (n_components == 1)
4123 return this->values_quad[q_point];
4124 else
4125 {
4126 if (n_components == dim &&
4127 this->data->element_type ==
4128 internal::MatrixFreeFunctions::ElementType::tensor_raviart_thomas)
4129 {
4130 // Piola transform is required
4131 if constexpr (running_in_debug_mode())
4132 {
4133 Assert(this->values_quad_initialized == true,
4134 internal::ExcAccessToUninitializedField());
4135 }
4136
4137 AssertIndexRange(q_point, this->n_quadrature_points);
4138 Assert(this->J_value != nullptr,
4139 internal::ExcMatrixFreeAccessToUninitializedMappingField(
4140 "update_values"));
4141 const std::size_t nqp = this->n_quadrature_points;
4142 Tensor<1, n_components, VectorizedArrayType> value_out;
4143
4144 if (!is_face &&
4145 this->cell_type == internal::MatrixFreeFunctions::cartesian)
4146 {
4147 // Cartesian cell
4148 const Tensor<2, dim, VectorizedArrayType> jac = this->jacobian[1];
4149 const VectorizedArrayType inv_det =
4150 (dim == 2) ? this->jacobian[0][0][0] * this->jacobian[0][1][1] :
4151 this->jacobian[0][0][0] * this->jacobian[0][1][1] *
4152 this->jacobian[0][2][2];
4153
4154 // J * u * det(J^-1)
4155 for (unsigned int comp = 0; comp < n_components; ++comp)
4156 value_out[comp] = this->values_quad[comp * nqp + q_point] *
4157 jac[comp][comp] * inv_det;
4158 }
4159 else
4160 {
4161 // Affine or general cell
4162 const Tensor<2, dim, VectorizedArrayType> inv_t_jac =
4163 (this->cell_type > internal::MatrixFreeFunctions::affine) ?
4164 this->jacobian[q_point] :
4165 this->jacobian[0];
4166 const Tensor<2, dim, VectorizedArrayType> jac =
4167 (this->cell_type > internal::MatrixFreeFunctions::affine) ?
4168 transpose(invert(inv_t_jac)) :
4169 this->jacobian[1];
4170
4171 // Derivatives are reordered for faces. Need to take this into
4172 // account
4173 const VectorizedArrayType inv_det =
4174 (is_face && dim == 2 && this->get_face_no() < 2) ?
4175 -determinant(inv_t_jac) :
4176 determinant(inv_t_jac);
4177 // J * u * det(J^-1)
4178 for (unsigned int comp = 0; comp < n_components; ++comp)
4179 {
4180 value_out[comp] = this->values_quad[q_point] * jac[comp][0];
4181 for (unsigned int e = 1; e < dim; ++e)
4182 value_out[comp] +=
4183 this->values_quad[e * nqp + q_point] * jac[comp][e];
4184 value_out[comp] *= inv_det;
4185 }
4186 }
4187 return value_out;
4188 }
4189 else
4190 {
4191 const std::size_t nqp = this->n_quadrature_points;
4192 Tensor<1, n_components_, VectorizedArrayType> return_value;
4193 for (unsigned int comp = 0; comp < n_components; ++comp)
4194 return_value[comp] = this->values_quad[comp * nqp + q_point];
4195 return return_value;
4196 }
4197 }
4198 }
4199
4200
4201
4202 template <int dim,
4203 int n_components_,
4204 typename Number,
4205 bool is_face,
4206 typename VectorizedArrayType>
4207 inline DEAL_II_ALWAYS_INLINE
4208 typename FEEvaluationBase<dim,
4209 n_components_,
4210 Number,
4211 is_face,
4212 VectorizedArrayType>::gradient_type
4213 FEEvaluationBase<dim, n_components_, Number, is_face, VectorizedArrayType>::
4214 get_gradient(const unsigned int q_point) const
4215 {
4216 if constexpr (running_in_debug_mode())
4217 {
4218 Assert(this->gradients_quad_initialized == true,
4219 internal::ExcAccessToUninitializedField());
4220 }
4221
4222 AssertIndexRange(q_point, this->n_quadrature_points);
4223 Assert(this->jacobian != nullptr,
4224 internal::ExcMatrixFreeAccessToUninitializedMappingField(
4225 "update_gradients"));
4226 const std::size_t nqp = this->n_quadrature_points;
4227
4228 if constexpr (n_components == dim && dim > 1)
4229 {
4230 if (this->data->element_type ==
4231 internal::MatrixFreeFunctions::ElementType::tensor_raviart_thomas)
4232 {
4233 // Piola transform is required
4234 if constexpr (running_in_debug_mode())
4235 {
4236 Assert(this->gradients_quad_initialized == true,
4237 internal::ExcAccessToUninitializedField());
4238 }
4239
4240 AssertIndexRange(q_point, this->n_quadrature_points);
4241 Assert(this->jacobian != nullptr,
4242 internal::ExcMatrixFreeAccessToUninitializedMappingField(
4243 "update_gradients"));
4244 const std::size_t nqp = this->n_quadrature_points;
4245 const std::size_t nqp_d = nqp * dim;
4246 Tensor<1, dim, Tensor<1, dim, VectorizedArrayType>> grad_out;
4247 const VectorizedArrayType *gradients =
4248 this->gradients_quad + q_point * dim;
4249
4250
4251 if (!is_face &&
4252 this->cell_type == internal::MatrixFreeFunctions::cartesian)
4253 {
4254 // Cartesian cell
4255 const Tensor<2, dim, VectorizedArrayType> &inv_t_jac =
4256 this->jacobian[0];
4257 const Tensor<2, dim, VectorizedArrayType> &jac =
4258 this->jacobian[1];
4259 const VectorizedArrayType inv_det =
4260 (dim == 2) ? this->jacobian[0][0][0] * this->jacobian[0][1][1] :
4261 this->jacobian[0][0][0] * this->jacobian[0][1][1] *
4262 this->jacobian[0][2][2];
4263
4264 // J * grad_quad * J^-1 * det(J^-1)
4265 for (unsigned int d = 0; d < dim; ++d)
4266 for (unsigned int comp = 0; comp < n_components; ++comp)
4267 grad_out[comp][d] = gradients[comp * nqp_d + d] *
4268 inv_t_jac[d][d] *
4269 (jac[comp][comp] * inv_det);
4270 }
4271 else if (this->cell_type <= internal::MatrixFreeFunctions::affine)
4272 {
4273 // Affine cell
4274 const Tensor<2, dim, VectorizedArrayType> &inv_t_jac =
4275 this->jacobian[0];
4276 const Tensor<2, dim, VectorizedArrayType> &jac =
4277 this->jacobian[1];
4278
4279 // Derivatives are reordered for faces. Need to take this into
4280 // account
4281 const VectorizedArrayType inv_det =
4282 (is_face && dim == 2 && this->get_face_no() < 2) ?
4283 -determinant(inv_t_jac) :
4284 determinant(inv_t_jac);
4285
4286 VectorizedArrayType tmp[dim][dim];
4287 // J * grad_quad * J^-1 * det(J^-1)
4288 for (unsigned int d = 0; d < dim; ++d)
4289 for (unsigned int e = 0; e < dim; ++e)
4290 {
4291 tmp[d][e] = inv_t_jac[d][0] * gradients[e * nqp_d + 0];
4292 for (unsigned int f = 1; f < dim; ++f)
4293 tmp[d][e] += inv_t_jac[d][f] * gradients[e * nqp_d + f];
4294 }
4295 for (unsigned int comp = 0; comp < n_components; ++comp)
4296 for (unsigned int d = 0; d < dim; ++d)
4297 {
4298 VectorizedArrayType res = jac[comp][0] * tmp[d][0];
4299 for (unsigned int f = 1; f < dim; ++f)
4300 res += jac[comp][f] * tmp[d][f];
4301
4302 grad_out[comp][d] = res * inv_det;
4303 }
4304 }
4305 else
4306 {
4307 // General cell
4308
4309 // This assert could be removed if we make sure that this is
4310 // updated even though update_hessians or update_jacobian_grads is
4311 // not passed, i.e make the necessary changes in
4312 // MatrixFreeFunctions::MappingInfoStorage::compute_update_flags
4313 Assert(this->jacobian_gradients_non_inverse != nullptr,
4314 internal::ExcMatrixFreeAccessToUninitializedMappingField(
4315 "update_hessians"));
4316
4317 const auto jac_grad =
4318 this->jacobian_gradients_non_inverse[q_point];
4319 const Tensor<2, dim, VectorizedArrayType> inv_t_jac =
4320 this->jacobian[q_point];
4321
4322 // Derivatives are reordered for faces. Need to take this into
4323 // account
4324 const VectorizedArrayType inv_det =
4325 (is_face && dim == 2 && this->get_face_no() < 2) ?
4326 -determinant(inv_t_jac) :
4327 determinant(inv_t_jac);
4328 const Tensor<2, dim, VectorizedArrayType> t_jac =
4329 invert(inv_t_jac);
4330
4331 // (J * grad_quad) * J^-1 * det(J^-1), part in braces
4332 VectorizedArrayType tmp[dim][dim];
4333 for (unsigned int d = 0; d < dim; ++d)
4334 for (unsigned int e = 0; e < dim; ++e)
4335 {
4336 tmp[e][d] = t_jac[0][d] * gradients[0 * nqp_d + e];
4337 for (unsigned int f = 1; f < dim; ++f)
4338 tmp[e][d] += t_jac[f][d] * gradients[f * nqp_d + e];
4339 }
4340
4341 // Add (jac_grad * values) * J^{-1} * det(J^{-1}), combine terms
4342 // outside braces with gradient part from above
4343 for (unsigned int d = 0; d < dim; ++d)
4344 {
4345 for (unsigned int e = 0; e < dim; ++e)
4346 tmp[e][d] +=
4347 jac_grad[e][d] * this->values_quad[e * nqp + q_point];
4348 for (unsigned int f = 0, r = dim; f < dim; ++f)
4349 for (unsigned int k = f + 1; k < dim; ++k, ++r)
4350 {
4351 tmp[k][d] +=
4352 jac_grad[r][d] * this->values_quad[f * nqp + q_point];
4353 tmp[f][d] +=
4354 jac_grad[r][d] * this->values_quad[k * nqp + q_point];
4355 }
4356 }
4357
4358 // Apply J^{-1} appearing in both terms outside braces above
4359 for (unsigned int d = 0; d < dim; ++d)
4360 for (unsigned int e = 0; e < dim; ++e)
4361 {
4362 VectorizedArrayType res = tmp[0][d] * inv_t_jac[e][0];
4363 for (unsigned int f = 1; f < dim; ++f)
4364 res += tmp[f][d] * inv_t_jac[e][f];
4365 grad_out[d][e] = res;
4366 }
4367
4368 // Add -(J^{-T} * jac_grad * J^{-1} * J * values * det(J^{-1})),
4369 // which can be expressed as a rank-1 update tmp[d] * tmp4[e],
4370 // where tmp = J * values and tmp4 = (J^{-T} * jac_grad * J^{-1})
4371 VectorizedArrayType tmp3[dim], tmp4[dim];
4372 for (unsigned int d = 0; d < dim; ++d)
4373 {
4374 tmp3[d] = inv_t_jac[0][d] * jac_grad[d][0];
4375 for (unsigned int e = 1; e < dim; ++e)
4376 tmp3[d] += inv_t_jac[e][d] * jac_grad[d][e];
4377 }
4378 for (unsigned int e = 0, k = dim; e < dim; ++e)
4379 for (unsigned int f = e + 1; f < dim; ++k, ++f)
4380 for (unsigned int d = 0; d < dim; ++d)
4381 {
4382 tmp3[f] += inv_t_jac[d][e] * jac_grad[k][d];
4383 tmp3[e] += inv_t_jac[d][f] * jac_grad[k][d];
4384 }
4385 for (unsigned int d = 0; d < dim; ++d)
4386 {
4387 tmp4[d] = tmp3[0] * inv_t_jac[d][0];
4388 for (unsigned int e = 1; e < dim; ++e)
4389 tmp4[d] += tmp3[e] * inv_t_jac[d][e];
4390 }
4391
4392 VectorizedArrayType tmp2[dim];
4393 for (unsigned int d = 0; d < dim; ++d)
4394 {
4395 tmp2[d] = t_jac[0][d] * this->values_quad[q_point];
4396 for (unsigned e = 1; e < dim; ++e)
4397 tmp2[d] +=
4398 t_jac[e][d] * this->values_quad[e * nqp + q_point];
4399 }
4400
4401 for (unsigned int d = 0; d < dim; ++d)
4402 for (unsigned int e = 0; e < dim; ++e)
4403 {
4404 grad_out[d][e] -= tmp4[e] * tmp2[d];
4405
4406 // finally multiply by det(J^{-1}) necessary in all
4407 // contributions above
4408 grad_out[d][e] *= inv_det;
4409 }
4410 }
4411 return grad_out;
4412 }
4413 }
4414 Tensor<1, n_components_, Tensor<1, dim, VectorizedArrayType>> grad_out;
4415
4416 // Cartesian cell
4417 if (!is_face && this->cell_type == internal::MatrixFreeFunctions::cartesian)
4418 {
4419 for (unsigned int comp = 0; comp < n_components; ++comp)
4420 for (unsigned int d = 0; d < dim; ++d)
4421 grad_out[comp][d] =
4422 this->gradients_quad[(comp * nqp + q_point) * dim + d] *
4423 this->jacobian[0][d][d];
4424 }
4425 // cell with general/affine Jacobian
4426 else
4427 {
4428 const Tensor<2, dim, VectorizedArrayType> &jac =
4429 this->jacobian[this->cell_type > internal::MatrixFreeFunctions::affine ?
4430 q_point :
4431 0];
4432 for (unsigned int comp = 0; comp < n_components; ++comp)
4433 for (unsigned int d = 0; d < dim; ++d)
4434 {
4435 grad_out[comp][d] =
4436 jac[d][0] * this->gradients_quad[(comp * nqp + q_point) * dim];
4437 for (unsigned int e = 1; e < dim; ++e)
4438 grad_out[comp][d] +=
4439 jac[d][e] *
4440 this->gradients_quad[(comp * nqp + q_point) * dim + e];
4441 }
4442 }
4443 if constexpr (n_components == 1)
4444 return grad_out[0];
4445 else
4446 return grad_out;
4447 }
4448
4449
4450
4451 template <int dim,
4452 int n_components_,
4453 typename Number,
4454 bool is_face,
4455 typename VectorizedArrayType>
4456 inline DEAL_II_ALWAYS_INLINE
4457 typename FEEvaluationBase<dim,
4458 n_components_,
4459 Number,
4460 is_face,
4461 VectorizedArrayType>::value_type
4462 FEEvaluationBase<dim, n_components_, Number, is_face, VectorizedArrayType>::
4463 get_normal_derivative(const unsigned int q_point) const
4464 {
4465 AssertIndexRange(q_point, this->n_quadrature_points);
4466 if constexpr (running_in_debug_mode())
4467 {
4468 Assert(this->gradients_quad_initialized == true,
4469 internal::ExcAccessToUninitializedField());
4470 }
4471
4472 Assert(this->normal_x_jacobian != nullptr,
4473 internal::ExcMatrixFreeAccessToUninitializedMappingField(
4474 "update_gradients"));
4475
4476 const std::size_t nqp = this->n_quadrature_points;
4477 Tensor<1, n_components, VectorizedArrayType> grad_out;
4478
4479 if (this->cell_type == internal::MatrixFreeFunctions::cartesian)
4480 for (unsigned int comp = 0; comp < n_components; ++comp)
4481 grad_out[comp] =
4482 this->gradients_quad[(comp * nqp + q_point) * dim + dim - 1] *
4483 (this->normal_x_jacobian[0][dim - 1]);
4484 else
4485 {
4486 const std::size_t index =
4487 this->cell_type <= internal::MatrixFreeFunctions::affine ? 0 : q_point;
4488 for (unsigned int comp = 0; comp < n_components; ++comp)
4489 {
4490 grad_out[comp] = this->gradients_quad[(comp * nqp + q_point) * dim] *
4491 this->normal_x_jacobian[index][0];
4492 for (unsigned int d = 1; d < dim; ++d)
4493 grad_out[comp] +=
4494 this->gradients_quad[(comp * nqp + q_point) * dim + d] *
4495 this->normal_x_jacobian[index][d];
4496 }
4497 }
4498 if constexpr (n_components == 1)
4499 return grad_out[0];
4500 else
4501 return grad_out;
4502 }
4503
4504
4505
4506 namespace internal
4507 {
4508 // compute tmp = hess_unit(u) * J^T. do this manually because we do not
4509 // store the lower diagonal because of symmetry
4510 template <typename VectorizedArrayType>
4511 inline void
4512 hessian_unit_times_jac(const Tensor<2, 1, VectorizedArrayType> &jac,
4513 const VectorizedArrayType *const hessians,
4514 const unsigned int,
4515 VectorizedArrayType (&tmp)[1][1])
4516 {
4517 tmp[0][0] = jac[0][0] * hessians[0];
4518 }
4519
4520 template <typename VectorizedArrayType>
4521 inline void
4522 hessian_unit_times_jac(const Tensor<2, 2, VectorizedArrayType> &jac,
4523 const VectorizedArrayType *const hessians,
4524 const unsigned int nqp,
4525 VectorizedArrayType (&tmp)[2][2])
4526 {
4527 for (unsigned int d = 0; d < 2; ++d)
4528 {
4529 tmp[0][d] = (jac[d][0] * hessians[0] + jac[d][1] * hessians[2 * nqp]);
4530 tmp[1][d] =
4531 (jac[d][0] * hessians[2 * nqp] + jac[d][1] * hessians[1 * nqp]);
4532 }
4533 }
4534
4535 template <typename VectorizedArrayType>
4536 inline void
4537 hessian_unit_times_jac(const Tensor<2, 3, VectorizedArrayType> &jac,
4538 const VectorizedArrayType *const hessians,
4539 const unsigned int nqp,
4540 VectorizedArrayType (&tmp)[3][3])
4541 {
4542 for (unsigned int d = 0; d < 3; ++d)
4543 {
4544 tmp[0][d] =
4545 (jac[d][0] * hessians[0 * nqp] + jac[d][1] * hessians[3 * nqp] +
4546 jac[d][2] * hessians[4 * nqp]);
4547 tmp[1][d] =
4548 (jac[d][0] * hessians[3 * nqp] + jac[d][1] * hessians[1 * nqp] +
4549 jac[d][2] * hessians[5 * nqp]);
4550 tmp[2][d] =
4551 (jac[d][0] * hessians[4 * nqp] + jac[d][1] * hessians[5 * nqp] +
4552 jac[d][2] * hessians[2 * nqp]);
4553 }
4554 }
4555 } // namespace internal
4556
4557
4558
4559 template <int dim,
4560 int n_components_,
4561 typename Number,
4562 bool is_face,
4563 typename VectorizedArrayType>
4564 inline typename FEEvaluationBase<dim,
4565 n_components_,
4566 Number,
4567 is_face,
4568 VectorizedArrayType>::hessian_type
4569 FEEvaluationBase<dim, n_components_, Number, is_face, VectorizedArrayType>::
4570 get_hessian(const unsigned int q_point) const
4571 {
4572 if constexpr (running_in_debug_mode())
4573 {
4574 Assert(this->hessians_quad_initialized == true,
4575 internal::ExcAccessToUninitializedField());
4576 }
4577 AssertIndexRange(q_point, this->n_quadrature_points);
4578
4579 Assert(this->jacobian != nullptr,
4580 internal::ExcMatrixFreeAccessToUninitializedMappingField(
4581 "update_hessian"));
4582 const Tensor<2, dim, VectorizedArrayType> &jac =
4583 this->jacobian[this->cell_type <= internal::MatrixFreeFunctions::affine ?
4584 0 :
4585 q_point];
4586
4587 Tensor<1, n_components, Tensor<2, dim, VectorizedArrayType>> hessian_out;
4588
4589 const std::size_t nqp = this->n_quadrature_points;
4590 constexpr unsigned int hdim = (dim * (dim + 1)) / 2;
4591
4592 // Cartesian cell
4593 if (!is_face && this->cell_type == internal::MatrixFreeFunctions::cartesian)
4594 {
4595 for (unsigned int comp = 0; comp < n_components; ++comp)
4596 {
4597 for (unsigned int d = 0; d < dim; ++d)
4598 hessian_out[comp][d][d] =
4599 this->hessians_quad[(comp * hdim + d) * nqp + q_point] *
4600 (jac[d][d] * jac[d][d]);
4601 switch (dim)
4602 {
4603 case 1:
4604 break;
4605 case 2:
4606 hessian_out[comp][0][1] =
4607 this->hessians_quad[(comp * hdim + 2) * nqp + q_point] *
4608 (jac[0][0] * jac[1][1]);
4609 break;
4610 case 3:
4611 hessian_out[comp][0][1] =
4612 this->hessians_quad[(comp * hdim + 3) * nqp + q_point] *
4613 (jac[0][0] * jac[1][1]);
4614 hessian_out[comp][0][2] =
4615 this->hessians_quad[(comp * hdim + 4) * nqp + q_point] *
4616 (jac[0][0] * jac[2][2]);
4617 hessian_out[comp][1][2] =
4618 this->hessians_quad[(comp * hdim + 5) * nqp + q_point] *
4619 (jac[1][1] * jac[2][2]);
4620 break;
4621 default:
4622 DEAL_II_NOT_IMPLEMENTED();
4623 }
4624 for (unsigned int d = 0; d < dim; ++d)
4625 for (unsigned int e = d + 1; e < dim; ++e)
4626 hessian_out[comp][e][d] = hessian_out[comp][d][e];
4627 }
4628 }
4629 // cell with general Jacobian, but constant within the cell
4630 else if (this->cell_type <= internal::MatrixFreeFunctions::affine)
4631 {
4632 for (unsigned int comp = 0; comp < n_components; ++comp)
4633 {
4634 VectorizedArrayType tmp[dim][dim];
4635 internal::hessian_unit_times_jac(
4636 jac, this->hessians_quad + comp * hdim * nqp + q_point, nqp, tmp);
4637
4638 // compute first part of hessian, J * tmp = J * hess_unit(u) * J^T
4639 for (unsigned int d = 0; d < dim; ++d)
4640 for (unsigned int e = d; e < dim; ++e)
4641 {
4642 hessian_out[comp][d][e] = jac[d][0] * tmp[0][e];
4643 for (unsigned int f = 1; f < dim; ++f)
4644 hessian_out[comp][d][e] += jac[d][f] * tmp[f][e];
4645 }
4646
4647 // no J' * grad(u) part here because the Jacobian is constant
4648 // throughout the cell and hence, its derivative is zero
4649
4650 // take symmetric part
4651 for (unsigned int d = 0; d < dim; ++d)
4652 for (unsigned int e = d + 1; e < dim; ++e)
4653 hessian_out[comp][e][d] = hessian_out[comp][d][e];
4654 }
4655 }
4656 // cell with general Jacobian
4657 else
4658 {
4659 const auto &jac_grad = this->jacobian_gradients[q_point];
4660 for (unsigned int comp = 0; comp < n_components; ++comp)
4661 {
4662 VectorizedArrayType tmp[dim][dim];
4663 internal::hessian_unit_times_jac(
4664 jac, this->hessians_quad + comp * hdim * nqp + q_point, nqp, tmp);
4665
4666 // compute first part of hessian, J * tmp = J * hess_unit(u) * J^T
4667 for (unsigned int d = 0; d < dim; ++d)
4668 for (unsigned int e = d; e < dim; ++e)
4669 {
4670 hessian_out[comp][d][e] = jac[d][0] * tmp[0][e];
4671 for (unsigned int f = 1; f < dim; ++f)
4672 hessian_out[comp][d][e] += jac[d][f] * tmp[f][e];
4673 }
4674
4675 // add diagonal part of J' * grad(u)
4676 for (unsigned int d = 0; d < dim; ++d)
4677 for (unsigned int e = 0; e < dim; ++e)
4678 hessian_out[comp][d][d] +=
4679 jac_grad[d][e] *
4680 this->gradients_quad[(comp * nqp + q_point) * dim + e];
4681
4682 // add off-diagonal part of J' * grad(u)
4683 for (unsigned int d = 0, count = dim; d < dim; ++d)
4684 for (unsigned int e = d + 1; e < dim; ++e, ++count)
4685 for (unsigned int f = 0; f < dim; ++f)
4686 hessian_out[comp][d][e] +=
4687 jac_grad[count][f] *
4688 this->gradients_quad[(comp * nqp + q_point) * dim + f];
4689
4690 // take symmetric part
4691 for (unsigned int d = 0; d < dim; ++d)
4692 for (unsigned int e = d + 1; e < dim; ++e)
4693 hessian_out[comp][e][d] = hessian_out[comp][d][e];
4694 }
4695 }
4696 if constexpr (n_components == 1)
4697 return hessian_out[0];
4698 else
4699 return hessian_out;
4700 }
4701
4702
4703
4704 template <int dim,
4705 int n_components_,
4706 typename Number,
4707 bool is_face,
4708 typename VectorizedArrayType>
4709 inline typename FEEvaluationBase<dim,
4710 n_components_,
4711 Number,
4712 is_face,
4713 VectorizedArrayType>::gradient_type
4714 FEEvaluationBase<dim, n_components_, Number, is_face, VectorizedArrayType>::
4715 get_hessian_diagonal(const unsigned int q_point) const
4716 {
4717 Assert(!is_face, ExcNotImplemented());
4718 if constexpr (running_in_debug_mode())
4719 {
4720 Assert(this->hessians_quad_initialized == true,
4721 internal::ExcAccessToUninitializedField());
4722 }
4723 AssertIndexRange(q_point, this->n_quadrature_points);
4724
4725 Assert(this->jacobian != nullptr, ExcNotImplemented());
4726 const Tensor<2, dim, VectorizedArrayType> &jac =
4727 this->jacobian[this->cell_type <= internal::MatrixFreeFunctions::affine ?
4728 0 :
4729 q_point];
4730
4731 const std::size_t nqp = this->n_quadrature_points;
4732 constexpr unsigned int hdim = (dim * (dim + 1)) / 2;
4733 Tensor<1, n_components_, Tensor<1, dim, VectorizedArrayType>> hessian_out;
4734
4735 // Cartesian cell
4736 if (this->cell_type == internal::MatrixFreeFunctions::cartesian)
4737 {
4738 for (unsigned int comp = 0; comp < n_components; ++comp)
4739 for (unsigned int d = 0; d < dim; ++d)
4740 hessian_out[comp][d] =
4741 this->hessians_quad[(comp * hdim + d) * nqp + q_point] *
4742 (jac[d][d] * jac[d][d]);
4743 }
4744 // cell with general Jacobian, but constant within the cell
4745 else if (this->cell_type == internal::MatrixFreeFunctions::affine)
4746 {
4747 for (unsigned int comp = 0; comp < n_components; ++comp)
4748 {
4749 // compute laplacian before the gradient because it needs to access
4750 // unscaled gradient data
4751 VectorizedArrayType tmp[dim][dim];
4752 internal::hessian_unit_times_jac(
4753 jac, this->hessians_quad + comp * hdim * nqp + q_point, nqp, tmp);
4754
4755 // compute only the trace part of hessian, J * tmp = J *
4756 // hess_unit(u) * J^T
4757 for (unsigned int d = 0; d < dim; ++d)
4758 {
4759 hessian_out[comp][d] = jac[d][0] * tmp[0][d];
4760 for (unsigned int f = 1; f < dim; ++f)
4761 hessian_out[comp][d] += jac[d][f] * tmp[f][d];
4762 }
4763 }
4764 }
4765 // cell with general Jacobian
4766 else
4767 {
4768 const auto &jac_grad = this->jacobian_gradients[q_point];
4769 for (unsigned int comp = 0; comp < n_components; ++comp)
4770 {
4771 // compute laplacian before the gradient because it needs to access
4772 // unscaled gradient data
4773 VectorizedArrayType tmp[dim][dim];
4774 internal::hessian_unit_times_jac(
4775 jac, this->hessians_quad + comp * hdim * nqp + q_point, nqp, tmp);
4776
4777 // compute only the trace part of hessian, J * tmp = J *
4778 // hess_unit(u) * J^T
4779 for (unsigned int d = 0; d < dim; ++d)
4780 {
4781 hessian_out[comp][d] = jac[d][0] * tmp[0][d];
4782 for (unsigned int f = 1; f < dim; ++f)
4783 hessian_out[comp][d] += jac[d][f] * tmp[f][d];
4784 }
4785
4786 for (unsigned int d = 0; d < dim; ++d)
4787 for (unsigned int e = 0; e < dim; ++e)
4788 hessian_out[comp][d] +=
4789 jac_grad[d][e] *
4790 this->gradients_quad[(comp * nqp + q_point) * dim + e];
4791 }
4792 }
4793
4794 if constexpr (n_components == 1)
4795 return hessian_out[0];
4796 else
4797 return hessian_out;
4798 }
4799
4800
4801
4802 template <int dim,
4803 int n_components_,
4804 typename Number,
4805 bool is_face,
4806 typename VectorizedArrayType>
4807 inline typename FEEvaluationBase<dim,
4808 n_components_,
4809 Number,
4810 is_face,
4811 VectorizedArrayType>::value_type
4812 FEEvaluationBase<dim, n_components_, Number, is_face, VectorizedArrayType>::
4813 get_laplacian(const unsigned int q_point) const
4814 {
4815 Assert(is_face == false, ExcNotImplemented());
4816 if constexpr (running_in_debug_mode())
4817 {
4818 Assert(this->hessians_quad_initialized == true,
4819 internal::ExcAccessToUninitializedField());
4820 }
4821 AssertIndexRange(q_point, this->n_quadrature_points);
4822
4823 const gradient_type hess_diag = get_hessian_diagonal(q_point);
4824 if constexpr (n_components == 1)
4825 {
4826 VectorizedArrayType sum = hess_diag[0];
4827 for (unsigned int d = 1; d < dim; ++d)
4828 sum += hess_diag[d];
4829 return sum;
4830 }
4831 else
4832 {
4833 Tensor<1, n_components_, VectorizedArrayType> laplacian_out;
4834 for (unsigned int comp = 0; comp < n_components; ++comp)
4835 {
4836 laplacian_out[comp] = hess_diag[comp][0];
4837 for (unsigned int d = 1; d < dim; ++d)
4838 laplacian_out[comp] += hess_diag[comp][d];
4839 }
4840 return laplacian_out;
4841 }
4842 }
4843
4844
4845
4846 template <int dim,
4847 int n_components_,
4848 typename Number,
4849 bool is_face,
4850 typename VectorizedArrayType>
4851 inline typename FEEvaluationBase<dim,
4852 n_components_,
4853 Number,
4854 is_face,
4855 VectorizedArrayType>::value_type
4856 FEEvaluationBase<dim, n_components_, Number, is_face, VectorizedArrayType>::
4857 get_normal_hessian(const unsigned int q_point) const
4858 {
4859 if constexpr (running_in_debug_mode())
4860 {
4861 Assert(this->hessians_quad_initialized == true,
4862 internal::ExcAccessToUninitializedField());
4863 }
4864 AssertIndexRange(q_point, this->n_quadrature_points);
4865
4866 Assert(this->normal_x_jacobian != nullptr,
4867 internal::ExcMatrixFreeAccessToUninitializedMappingField(
4868 "update_hessians"));
4869
4870 Tensor<1, n_components, VectorizedArrayType> hessian_out;
4871
4872 const std::size_t nqp = this->n_quadrature_points;
4873 constexpr unsigned int hdim = (dim * (dim + 1)) / 2;
4874
4875 if (this->cell_type <= internal::MatrixFreeFunctions::affine)
4876 {
4877 const auto nxj = this->normal_x_jacobian[0];
4878
4879 for (unsigned int comp = 0; comp < n_components; ++comp)
4880 {
4881 for (unsigned int d = 0; d < dim; ++d)
4882 hessian_out[comp] +=
4883 this->hessians_quad[(comp * hdim + d) * nqp + q_point] *
4884 (nxj[d]) * (nxj[d]);
4885
4886 switch (dim)
4887 {
4888 case 1:
4889 break;
4890 case 2:
4891 hessian_out[comp] +=
4892 this->hessians_quad[(comp * hdim + 2) * nqp + q_point] *
4893 (nxj[0] * nxj[1]);
4894 break;
4895 case 3:
4896 hessian_out[comp] +=
4897 2. * this->hessians_quad[(comp * hdim + 3) * nqp + q_point] *
4898 (nxj[0] * nxj[1]);
4899 hessian_out[comp] +=
4900 2. * this->hessians_quad[(comp * hdim + 4) * nqp + q_point] *
4901 (nxj[0] * nxj[2]);
4902 hessian_out[comp] +=
4903 2. * this->hessians_quad[(comp * hdim + 5) * nqp + q_point] *
4904 (nxj[1] * nxj[2]);
4905 break;
4906 default:
4907 DEAL_II_NOT_IMPLEMENTED();
4908 }
4909 }
4910 }
4911 // cell with general Jacobian
4912 else
4913 {
4914 const auto normal = this->normal_vector(q_point);
4915 const auto hessian = get_hessian(q_point);
4916
4917 if constexpr (n_components == 1)
4918 hessian_out[0] = hessian * normal * normal;
4919 else
4920 for (unsigned int comp = 0; comp < n_components; ++comp)
4921 hessian_out[comp] = hessian[comp] * normal * normal;
4922 }
4923 if constexpr (n_components == 1)
4924 return hessian_out[0];
4925 else
4926 return hessian_out;
4927 }
4928
4929
4930
4931 template <int dim,
4932 int n_components_,
4933 typename Number,
4934 bool is_face,
4935 typename VectorizedArrayType>
4936 inline DEAL_II_ALWAYS_INLINE void
4937 FEEvaluationBase<dim, n_components_, Number, is_face, VectorizedArrayType>::
4938 submit_dof_value(const value_type val_in, const unsigned int dof)
4939 {
4940 if constexpr (running_in_debug_mode())
4941 {
4942 this->dof_values_initialized = true;
4943 }
4944 const std::size_t dofs = this->data->dofs_per_component_on_cell;
4945 AssertIndexRange(dof, this->data->dofs_per_component_on_cell);
4946 for (unsigned int comp = 0; comp < n_components; ++comp)
4947 if constexpr (n_components == 1)
4948 this->values_dofs[comp * dofs + dof] = val_in;
4949 else
4950 this->values_dofs[comp * dofs + dof] = val_in[comp];
4951 }
4952
4953
4954
4955 template <int dim,
4956 int n_components_,
4957 typename Number,
4958 bool is_face,
4959 typename VectorizedArrayType>
4960 inline DEAL_II_ALWAYS_INLINE void
4961 FEEvaluationBase<dim, n_components_, Number, is_face, VectorizedArrayType>::
4962 submit_value(const value_type val_in, const unsigned int q_point)
4963 {
4964 if constexpr (running_in_debug_mode())
4965 {
4966 Assert(this->is_reinitialized, ExcNotInitialized());
4967 }
4968 AssertIndexRange(q_point, this->n_quadrature_points);
4969 Assert(this->J_value != nullptr,
4970 internal::ExcMatrixFreeAccessToUninitializedMappingField(
4971 "update_values"));
4972 if constexpr (running_in_debug_mode())
4973 {
4974 this->values_quad_submitted = true;
4975 }
4976
4977 const std::size_t nqp = this->n_quadrature_points;
4978 VectorizedArrayType *values = this->values_quad + q_point;
4979
4980 const VectorizedArrayType JxW =
4981 this->cell_type <= internal::MatrixFreeFunctions::affine ?
4982 this->J_value[0] * this->quadrature_weights[q_point] :
4983 this->J_value[q_point];
4984 if constexpr (n_components == 1)
4985 values[0] = val_in * JxW;
4986 else
4987 {
4988 if (n_components == dim &&
4989 this->data->element_type ==
4990 internal::MatrixFreeFunctions::ElementType::tensor_raviart_thomas)
4991 {
4992 // Piola transform is required
4993 AssertIndexRange(q_point, this->n_quadrature_points);
4994 Assert(this->J_value != nullptr,
4995 internal::ExcMatrixFreeAccessToUninitializedMappingField(
4996 "update_value"));
4997 if constexpr (running_in_debug_mode())
4998 {
4999 Assert(this->is_reinitialized, ExcNotInitialized());
5000 this->values_quad_submitted = true;
5001 }
5002
5003 VectorizedArrayType *values = this->values_quad + q_point;
5004 const std::size_t nqp = this->n_quadrature_points;
5005
5006 if (!is_face &&
5007 this->cell_type == internal::MatrixFreeFunctions::cartesian)
5008 {
5009 const Tensor<2, dim, VectorizedArrayType> jac = this->jacobian[1];
5010 const VectorizedArrayType weight =
5011 this->quadrature_weights[q_point];
5012
5013 for (unsigned int comp = 0; comp < n_components; ++comp)
5014 values[comp * nqp] = val_in[comp] * weight * jac[comp][comp];
5015 }
5016 else
5017 {
5018 // Affine or general cell
5019 const Tensor<2, dim, VectorizedArrayType> inv_t_jac =
5020 (this->cell_type > internal::MatrixFreeFunctions::affine) ?
5021 this->jacobian[q_point] :
5022 this->jacobian[0];
5023
5024 // Derivatives are reordered for faces. Need to take this into
5025 // account and 1/inv_det != J_value for faces
5026 const VectorizedArrayType fac =
5027 (!is_face) ?
5028 this->quadrature_weights[q_point] :
5029 (((this->cell_type > internal::MatrixFreeFunctions::affine) ?
5030 this->J_value[q_point] :
5031 this->J_value[0] * this->quadrature_weights[q_point]) *
5032 ((dim == 2 && this->get_face_no() < 2) ?
5033 -determinant(inv_t_jac) :
5034 determinant(inv_t_jac)));
5035 const Tensor<2, dim, VectorizedArrayType> jac =
5036 (this->cell_type > internal::MatrixFreeFunctions::affine) ?
5037 transpose(invert(inv_t_jac)) :
5038 this->jacobian[1];
5039
5040 // J^T * u * factor
5041 for (unsigned int comp = 0; comp < n_components; ++comp)
5042 {
5043 values[comp * nqp] = val_in[0] * jac[0][comp];
5044 for (unsigned int e = 1; e < dim; ++e)
5045 values[comp * nqp] += val_in[e] * jac[e][comp];
5046 values[comp * nqp] *= fac;
5047 }
5048 }
5049 }
5050 else
5051 for (unsigned int comp = 0; comp < n_components; ++comp)
5052 values[comp * nqp] = val_in[comp] * JxW;
5053 }
5054 }
5055
5056
5057
5058 template <int dim,
5059 int n_components_,
5060 typename Number,
5061 bool is_face,
5062 typename VectorizedArrayType>
5063 template <int, typename>
5064 inline DEAL_II_ALWAYS_INLINE void
5065 FEEvaluationBase<dim, n_components_, Number, is_face, VectorizedArrayType>::
5066 submit_value(const Tensor<1, 1, VectorizedArrayType> val_in,
5067 const unsigned int q_point)
5068 {
5069 static_assert(n_components == 1,
5070 "Do not try to modify the default template parameters used for"
5071 " selectively enabling this function via std::enable_if!");
5072 submit_value(val_in[0], q_point);
5073 }
5074
5075
5076
5077 template <int dim,
5078 int n_components_,
5079 typename Number,
5080 bool is_face,
5081 typename VectorizedArrayType>
5082 inline DEAL_II_ALWAYS_INLINE void
5083 FEEvaluationBase<dim, n_components_, Number, is_face, VectorizedArrayType>::
5084 submit_gradient(const gradient_type grad_in, const unsigned int q_point)
5085 {
5086 if constexpr (running_in_debug_mode())
5087 {
5088 Assert(this->is_reinitialized, ExcNotInitialized());
5089 }
5090 AssertIndexRange(q_point, this->n_quadrature_points);
5091 Assert(this->J_value != nullptr,
5092 internal::ExcMatrixFreeAccessToUninitializedMappingField(
5093 "update_gradients"));
5094 Assert(this->jacobian != nullptr,
5095 internal::ExcMatrixFreeAccessToUninitializedMappingField(
5096 "update_gradients"));
5097 if constexpr (running_in_debug_mode())
5098 {
5099 this->gradients_quad_submitted = true;
5100 }
5101
5102 if constexpr (dim > 1 && n_components == dim)
5103 {
5104 if (this->data->element_type ==
5105 internal::MatrixFreeFunctions::ElementType::tensor_raviart_thomas)
5106 {
5107 // Piola transform is required
5108
5109 if constexpr (running_in_debug_mode())
5110 {
5111 Assert(this->is_reinitialized, ExcNotInitialized());
5112 }
5113 AssertIndexRange(q_point, this->n_quadrature_points);
5114 Assert(this->J_value != nullptr,
5115 internal::ExcMatrixFreeAccessToUninitializedMappingField(
5116 "update_gradients"));
5117 Assert(this->jacobian != nullptr,
5118 internal::ExcMatrixFreeAccessToUninitializedMappingField(
5119 "update_gradients"));
5120 if constexpr (running_in_debug_mode())
5121 {
5122 this->gradients_quad_submitted = true;
5123 }
5124
5125 VectorizedArrayType *gradients = this->gradients_quad + q_point * dim;
5126 VectorizedArrayType *values =
5127 this->values_from_gradients_quad + q_point;
5128 const std::size_t nqp = this->n_quadrature_points;
5129 const std::size_t nqp_d = nqp * dim;
5130
5131 if (!is_face &&
5132 this->cell_type == internal::MatrixFreeFunctions::cartesian)
5133 {
5134 // Cartesian cell
5135 const Tensor<2, dim, VectorizedArrayType> &inv_t_jac =
5136 this->jacobian[0];
5137 const Tensor<2, dim, VectorizedArrayType> &jac =
5138 this->jacobian[1];
5139 const VectorizedArrayType weight =
5140 this->quadrature_weights[q_point];
5141 for (unsigned int d = 0; d < dim; ++d)
5142 for (unsigned int comp = 0; comp < n_components; ++comp)
5143 gradients[comp * nqp_d + d] = grad_in[comp][d] *
5144 inv_t_jac[d][d] *
5145 (jac[comp][comp] * weight);
5146 }
5147 else if (this->cell_type <= internal::MatrixFreeFunctions::affine)
5148 {
5149 // Affine cell
5150 const Tensor<2, dim, VectorizedArrayType> &inv_t_jac =
5151 this->jacobian[0];
5152 const Tensor<2, dim, VectorizedArrayType> &jac =
5153 this->jacobian[1];
5154
5155 // Derivatives are reordered for faces. Need to take this into
5156 // account and 1/inv_det != J_value for faces
5157 const VectorizedArrayType fac =
5158 (!is_face) ?
5159 this->quadrature_weights[q_point] :
5160 this->J_value[0] * this->quadrature_weights[q_point] *
5161 ((dim == 2 && this->get_face_no() < 2) ?
5162 -determinant(inv_t_jac) :
5163 determinant(inv_t_jac));
5164
5165 // J_{j,i} * J^{-1}_{k,m} * grad_in_{j,m} * factor
5166 VectorizedArrayType tmp[dim][dim];
5167 for (unsigned int d = 0; d < dim; ++d)
5168 for (unsigned int e = 0; e < dim; ++e)
5169 {
5170 tmp[d][e] = inv_t_jac[0][d] * grad_in[e][0];
5171 for (unsigned int f = 1; f < dim; ++f)
5172 tmp[d][e] += inv_t_jac[f][d] * grad_in[e][f];
5173 }
5174 for (unsigned int comp = 0; comp < n_components; ++comp)
5175 for (unsigned int d = 0; d < dim; ++d)
5176 {
5177 VectorizedArrayType res = jac[0][comp] * tmp[d][0];
5178 for (unsigned int f = 1; f < dim; ++f)
5179 res += jac[f][comp] * tmp[d][f];
5180
5181 gradients[comp * nqp_d + d] = res * fac;
5182 }
5183 }
5184 else
5185 {
5186 // General cell
5187
5188 const auto jac_grad =
5189 this->jacobian_gradients_non_inverse[q_point];
5190 const Tensor<2, dim, VectorizedArrayType> inv_t_jac =
5191 this->jacobian[q_point];
5192
5193 // Derivatives are reordered for faces. Need to take this into
5194 // account and 1/inv_det != J_value for faces
5195 const VectorizedArrayType fac =
5196 (!is_face) ? this->quadrature_weights[q_point] :
5197 this->J_value[q_point] *
5198 ((dim == 2 && this->get_face_no() < 2) ?
5199 -determinant(inv_t_jac) :
5200 determinant(inv_t_jac));
5201 const Tensor<2, dim, VectorizedArrayType> t_jac =
5202 invert(inv_t_jac);
5203
5204 // Start evaluation for values part below to enable the compiler
5205 // to possibly re-use the same computation in get_gradient()
5206 // without interfering with stores to 'gradients'
5207 VectorizedArrayType tmp3[dim], tmp4[dim];
5208 for (unsigned int d = 0; d < dim; ++d)
5209 {
5210 tmp3[d] = inv_t_jac[0][d] * jac_grad[d][0];
5211 for (unsigned int e = 1; e < dim; ++e)
5212 tmp3[d] += inv_t_jac[e][d] * jac_grad[d][e];
5213 }
5214 for (unsigned int e = 0, k = dim; e < dim; ++e)
5215 for (unsigned int f = e + 1; f < dim; ++k, ++f)
5216 for (unsigned int d = 0; d < dim; ++d)
5217 {
5218 tmp3[f] += inv_t_jac[d][e] * jac_grad[k][d];
5219 tmp3[e] += inv_t_jac[d][f] * jac_grad[k][d];
5220 }
5221 for (unsigned int d = 0; d < dim; ++d)
5222 {
5223 tmp4[d] = tmp3[0] * inv_t_jac[d][0];
5224 for (unsigned int e = 1; e < dim; ++e)
5225 tmp4[d] += tmp3[e] * inv_t_jac[d][e];
5226 }
5227
5228 const Tensor<2, dim, VectorizedArrayType> grad_in_scaled =
5229 fac * grad_in;
5230
5231 VectorizedArrayType tmp[dim][dim];
5232
5233 // J * (J^{-1} * (grad_in * factor))
5234 for (unsigned int d = 0; d < dim; ++d)
5235 for (unsigned int e = 0; e < dim; ++e)
5236 {
5237 tmp[d][e] = inv_t_jac[0][d] * grad_in_scaled[e][0];
5238 for (unsigned int f = 1; f < dim; ++f)
5239 tmp[d][e] += inv_t_jac[f][d] * grad_in_scaled[e][f];
5240 }
5241
5242 for (unsigned int d = 0; d < dim; ++d)
5243 for (unsigned int e = 0; e < dim; ++e)
5244 {
5245 VectorizedArrayType res = t_jac[d][0] * tmp[e][0];
5246 for (unsigned int f = 1; f < dim; ++f)
5247 res += t_jac[d][f] * tmp[e][f];
5248
5249 gradients[d * nqp_d + e] = res;
5250 }
5251
5252 // jac_grad * (J^{-1} * (grad_in * factor)), re-use part in braces
5253 // as 'tmp' from above
5254 VectorizedArrayType value[dim];
5255 for (unsigned int d = 0; d < dim; ++d)
5256 {
5257 value[d] = tmp[d][0] * jac_grad[d][0];
5258 for (unsigned int e = 1; e < dim; ++e)
5259 value[d] += tmp[d][e] * jac_grad[d][e];
5260 }
5261 for (unsigned int e = 0, k = dim; e < dim; ++e)
5262 for (unsigned int f = e + 1; f < dim; ++k, ++f)
5263 for (unsigned int d = 0; d < dim; ++d)
5264 {
5265 value[e] += tmp[f][d] * jac_grad[k][d];
5266 value[f] += tmp[e][d] * jac_grad[k][d];
5267 }
5268
5269 // -(grad_in * factor) * J * (J^{-T} * jac_grad * J^{-1})
5270 // = -(grad_in * factor) * J * ( \------- tmp4 ---------/ )
5271 for (unsigned int d = 0; d < dim; ++d)
5272 {
5273 VectorizedArrayType tmp2 = grad_in_scaled[d][0] * tmp4[0];
5274 for (unsigned int e = 1; e < dim; ++e)
5275 tmp2 += grad_in_scaled[d][e] * tmp4[e];
5276 for (unsigned int e = 0; e < dim; ++e)
5277 value[e] -= t_jac[e][d] * tmp2;
5278 }
5279
5280 for (unsigned int d = 0; d < dim; ++d)
5281 values[d * nqp] = value[d];
5282 }
5283 return;
5284 }
5285 }
5286
5287 const std::size_t nqp_d = this->n_quadrature_points * dim;
5288 VectorizedArrayType *gradients = this->gradients_quad + q_point * dim;
5289
5290 if (!is_face && this->cell_type == internal::MatrixFreeFunctions::cartesian)
5291 {
5292 const VectorizedArrayType JxW =
5293 this->J_value[0] * this->quadrature_weights[q_point];
5294
5295 // Load all entries before starting to write back to make sure the
5296 // compiler sees opportunity of loads in a possibly nearby
5297 // get_gradient() function (i.e., the compiler should not think that
5298 // 'jacobian' could alias with 'gradients').
5299 std::array<VectorizedArrayType, dim> jac;
5300 for (unsigned int d = 0; d < dim; ++d)
5301 jac[d] = this->jacobian[0][d][d];
5302
5303 for (unsigned int d = 0; d < dim; ++d)
5304 {
5305 const VectorizedArrayType factor = this->jacobian[0][d][d] * JxW;
5306 if constexpr (n_components == 1)
5307 gradients[d] = grad_in[d] * factor;
5308 else
5309 for (unsigned int comp = 0; comp < n_components; ++comp)
5310 gradients[comp * nqp_d + d] = grad_in[comp][d] * factor;
5311 }
5312 }
5313 else
5314 {
5315 const Tensor<2, dim, VectorizedArrayType> jac =
5316 this->cell_type > internal::MatrixFreeFunctions::affine ?
5317 this->jacobian[q_point] :
5318 this->jacobian[0];
5319 const VectorizedArrayType JxW =
5320 this->cell_type > internal::MatrixFreeFunctions::affine ?
5321 this->J_value[q_point] :
5322 this->J_value[0] * this->quadrature_weights[q_point];
5323 if constexpr (n_components == 1)
5324 for (unsigned int d = 0; d < dim; ++d)
5325 {
5326 VectorizedArrayType new_val = jac[0][d] * grad_in[0];
5327 for (unsigned int e = 1; e < dim; ++e)
5328 new_val += (jac[e][d] * grad_in[e]);
5329 gradients[d] = new_val * JxW;
5330 }
5331 else
5332 for (unsigned int comp = 0; comp < n_components; ++comp)
5333 for (unsigned int d = 0; d < dim; ++d)
5334 {
5335 VectorizedArrayType new_val = jac[0][d] * grad_in[comp][0];
5336 for (unsigned int e = 1; e < dim; ++e)
5337 new_val += (jac[e][d] * grad_in[comp][e]);
5338 gradients[comp * nqp_d + d] = new_val * JxW;
5339 }
5340 }
5341 }
5342
5343
5344
5345 template <int dim,
5346 int n_components_,
5347 typename Number,
5348 bool is_face,
5349 typename VectorizedArrayType>
5350 template <int, typename>
5351 inline DEAL_II_ALWAYS_INLINE void
5352 FEEvaluationBase<dim, n_components_, Number, is_face, VectorizedArrayType>::
5353 submit_gradient(const Tensor<2, 1, VectorizedArrayType> grad_in,
5354 const unsigned int q_point)
5355 {
5356 static_assert(n_components == 1 && dim == 1,
5357 "Do not try to modify the default template parameters used for"
5358 " selectively enabling this function via std::enable_if!");
5359 submit_gradient(grad_in[0], q_point);
5360 }
5361
5362
5363
5364 template <int dim,
5365 int n_components_,
5366 typename Number,
5367 bool is_face,
5368 typename VectorizedArrayType>
5369 inline DEAL_II_ALWAYS_INLINE void
5370 FEEvaluationBase<dim, n_components_, Number, is_face, VectorizedArrayType>::
5371 submit_normal_derivative(const value_type grad_in, const unsigned int q_point)
5372 {
5373 AssertIndexRange(q_point, this->n_quadrature_points);
5374 Assert(this->normal_x_jacobian != nullptr,
5375 internal::ExcMatrixFreeAccessToUninitializedMappingField(
5376 "update_gradients"));
5377 if constexpr (running_in_debug_mode())
5378 {
5379 this->gradients_quad_submitted = true;
5380 }
5381
5382 const std::size_t nqp_d = this->n_quadrature_points * dim;
5383 VectorizedArrayType *gradients = this->gradients_quad + q_point * dim;
5384
5385 if (this->cell_type == internal::MatrixFreeFunctions::cartesian)
5386 {
5387 const VectorizedArrayType JxW_jac = this->J_value[0] *
5388 this->quadrature_weights[q_point] *
5389 this->normal_x_jacobian[0][dim - 1];
5390 for (unsigned int comp = 0; comp < n_components; ++comp)
5391 {
5392 for (unsigned int d = 0; d < dim - 1; ++d)
5393 gradients[comp * nqp_d + d] = VectorizedArrayType();
5394 if constexpr (n_components == 1)
5395 gradients[dim - 1] = grad_in * JxW_jac;
5396 else
5397 gradients[comp * nqp_d + dim - 1] = grad_in[comp] * JxW_jac;
5398 }
5399 }
5400 else
5401 {
5402 const unsigned int index =
5403 this->cell_type <= internal::MatrixFreeFunctions::affine ? 0 : q_point;
5404 const Tensor<1, dim, VectorizedArrayType> jac =
5405 this->normal_x_jacobian[index];
5406 const VectorizedArrayType JxW =
5407 (this->cell_type <= internal::MatrixFreeFunctions::affine) ?
5408 this->J_value[index] * this->quadrature_weights[q_point] :
5409 this->J_value[index];
5410 for (unsigned int comp = 0; comp < n_components; ++comp)
5411 for (unsigned int d = 0; d < dim; ++d)
5412 if constexpr (n_components == 1)
5413 gradients[d] = (grad_in * JxW) * jac[d];
5414 else
5415 gradients[comp * nqp_d + d] = (grad_in[comp] * JxW) * jac[d];
5416 }
5417 }
5418
5419
5420
5421 template <int dim,
5422 int n_components_,
5423 typename Number,
5424 bool is_face,
5425 typename VectorizedArrayType>
5426 inline DEAL_II_ALWAYS_INLINE void
5427 FEEvaluationBase<dim, n_components_, Number, is_face, VectorizedArrayType>::
5428 submit_hessian(const hessian_type hessian_in, const unsigned int q_point)
5429 {
5430 if constexpr (running_in_debug_mode())
5431 {
5432 Assert(this->is_reinitialized, ExcNotInitialized());
5433 }
5434 AssertIndexRange(q_point, this->n_quadrature_points);
5435 Assert(this->J_value != nullptr,
5436 internal::ExcMatrixFreeAccessToUninitializedMappingField(
5437 "update_hessians"));
5438 Assert(this->jacobian != nullptr,
5439 internal::ExcMatrixFreeAccessToUninitializedMappingField(
5440 "update_hessians"));
5441 if constexpr (running_in_debug_mode())
5442 {
5443 this->hessians_quad_submitted = true;
5444 }
5445
5446 // compute hessian_unit = J^T * hessian_in(u) * J
5447 const std::size_t nqp = this->n_quadrature_points;
5448 constexpr unsigned int hdim = (dim * (dim + 1)) / 2;
5449 if (!is_face && this->cell_type == internal::MatrixFreeFunctions::cartesian)
5450 {
5451 const VectorizedArrayType JxW =
5452 this->J_value[0] * this->quadrature_weights[q_point];
5453
5454 // diagonal part
5455 for (unsigned int d = 0; d < dim; ++d)
5456 {
5457 const auto jac_d = this->jacobian[0][d][d];
5458 const VectorizedArrayType factor = jac_d * jac_d * JxW;
5459 for (unsigned int comp = 0; comp < n_components; ++comp)
5460 if constexpr (n_components == 1)
5461 this->hessians_quad[d * nqp + q_point] =
5462 hessian_in[d][d] * factor;
5463 else
5464 this->hessians_quad[(comp * hdim + d) * nqp + q_point] =
5465 hessian_in[comp][d][d] * factor;
5466 }
5467
5468 // off diagonal part
5469 for (unsigned int d = 1, off_dia = dim; d < dim; ++d)
5470 for (unsigned int e = 0; e < d; ++e, ++off_dia)
5471 {
5472 const auto jac_d = this->jacobian[0][d][d];
5473 const auto jac_e = this->jacobian[0][e][e];
5474 const VectorizedArrayType factor = jac_d * jac_e * JxW;
5475 for (unsigned int comp = 0; comp < n_components; ++comp)
5476 if constexpr (n_components == 1)
5477 this->hessians_quad[off_dia * nqp + q_point] =
5478 (hessian_in[d][e] + hessian_in[e][d]) * factor;
5479 else
5480 this->hessians_quad[(comp * hdim + off_dia) * nqp + q_point] =
5481 (hessian_in[comp][d][e] + hessian_in[comp][e][d]) * factor;
5482 }
5483 }
5484 // cell with general Jacobian, but constant within the cell
5485 else if (this->cell_type <= internal::MatrixFreeFunctions::affine)
5486 {
5487 const Tensor<2, dim, VectorizedArrayType> jac = this->jacobian[0];
5488 const VectorizedArrayType JxW =
5489 this->J_value[0] * this->quadrature_weights[q_point];
5490 for (unsigned int comp = 0; comp < n_components; ++comp)
5491 {
5492 Tensor<2, dim, VectorizedArrayType> hessian_c;
5493 if constexpr (n_components == 1)
5494 hessian_c = hessian_in;
5495 else
5496 hessian_c = hessian_in[comp];
5497
5498 // 1. tmp = hessian(u) * J
5499 VectorizedArrayType tmp[dim][dim];
5500 for (unsigned int i = 0; i < dim; ++i)
5501 for (unsigned int j = 0; j < dim; ++j)
5502 {
5503 tmp[i][j] = hessian_c[i][0] * jac[0][j];
5504 for (unsigned int k = 1; k < dim; ++k)
5505 tmp[i][j] += hessian_c[i][k] * jac[k][j];
5506 }
5507
5508 // 2. hessian_unit = J^T * tmp
5509 VectorizedArrayType tmp2[dim][dim];
5510 for (unsigned int i = 0; i < dim; ++i)
5511 for (unsigned int j = 0; j < dim; ++j)
5512 {
5513 tmp2[i][j] = jac[0][i] * tmp[0][j];
5514 for (unsigned int k = 1; k < dim; ++k)
5515 tmp2[i][j] += jac[k][i] * tmp[k][j];
5516 }
5517
5518 // diagonal part
5519 for (unsigned int d = 0; d < dim; ++d)
5520 this->hessians_quad[(comp * hdim + d) * nqp + q_point] =
5521 tmp2[d][d] * JxW;
5522
5523 // off diagonal part
5524 for (unsigned int d = 0, off_diag = dim; d < dim; ++d)
5525 for (unsigned int e = d + 1; e < dim; ++e, ++off_diag)
5526 this->hessians_quad[(comp * hdim + off_diag) * nqp + q_point] =
5527 (tmp2[d][e] + tmp2[e][d]) * JxW;
5528 }
5529 }
5530 else
5531 {
5532 const Tensor<2, dim, VectorizedArrayType> jac = this->jacobian[q_point];
5533 const VectorizedArrayType JxW = this->J_value[q_point];
5534 const auto &jac_grad = this->jacobian_gradients[q_point];
5535 for (unsigned int comp = 0; comp < n_components; ++comp)
5536 {
5537 Tensor<2, dim, VectorizedArrayType> hessian_c;
5538 if constexpr (n_components == 1)
5539 hessian_c = hessian_in;
5540 else
5541 hessian_c = hessian_in[comp];
5542
5543 // 1. tmp = hessian(u) * J
5544 VectorizedArrayType tmp[dim][dim];
5545 for (unsigned int i = 0; i < dim; ++i)
5546 for (unsigned int j = 0; j < dim; ++j)
5547 {
5548 tmp[i][j] = hessian_c[i][0] * jac[0][j];
5549 for (unsigned int k = 1; k < dim; ++k)
5550 tmp[i][j] += hessian_c[i][k] * jac[k][j];
5551 }
5552
5553 // 2. hessian_unit = J^T * tmp
5554 VectorizedArrayType tmp2[dim][dim];
5555 for (unsigned int i = 0; i < dim; ++i)
5556 for (unsigned int j = 0; j < dim; ++j)
5557 {
5558 tmp2[i][j] = jac[0][i] * tmp[0][j];
5559 for (unsigned int k = 1; k < dim; ++k)
5560 tmp2[i][j] += jac[k][i] * tmp[k][j];
5561 }
5562
5563 // diagonal part
5564 for (unsigned int d = 0; d < dim; ++d)
5565 this->hessians_quad[(comp * hdim + d) * nqp + q_point] =
5566 tmp2[d][d] * JxW;
5567
5568 // off diagonal part
5569 for (unsigned int d = 0, off_diag = dim; d < dim; ++d)
5570 for (unsigned int e = d + 1; e < dim; ++e, ++off_diag)
5571 this->hessians_quad[(comp * hdim + off_diag) * nqp + q_point] =
5572 (tmp2[d][e] + tmp2[e][d]) * JxW;
5573
5574 // 3. gradient_unit = J' * hessian
5575 for (unsigned int d = 0; d < dim; ++d)
5576 {
5577 VectorizedArrayType sum = 0;
5578 for (unsigned int e = 0; e < dim; ++e)
5579 sum += hessian_c[e][e] * jac_grad[e][d];
5580 for (unsigned int e = 0, count = dim; e < dim; ++e)
5581 for (unsigned int f = e + 1; f < dim; ++f, ++count)
5582 sum +=
5583 (hessian_c[e][f] + hessian_c[f][e]) * jac_grad[count][d];
5584 this->gradients_from_hessians_quad[(comp * nqp + q_point) * dim +
5585 d] = sum * JxW;
5586 }
5587 }
5588 }
5589 }
5590
5591
5592
5593 template <int dim,
5594 int n_components_,
5595 typename Number,
5596 bool is_face,
5597 typename VectorizedArrayType>
5598 inline DEAL_II_ALWAYS_INLINE void
5599 FEEvaluationBase<dim, n_components_, Number, is_face, VectorizedArrayType>::
5600 submit_normal_hessian(const value_type normal_hessian_in,
5601 const unsigned int q_point)
5602 {
5603 if constexpr (running_in_debug_mode())
5604 {
5605 Assert(this->is_reinitialized, ExcNotInitialized());
5606 }
5607 AssertIndexRange(q_point, this->n_quadrature_points);
5608 Assert(this->J_value != nullptr,
5609 internal::ExcMatrixFreeAccessToUninitializedMappingField(
5610 "update_hessians"));
5611 Assert(this->jacobian != nullptr,
5612 internal::ExcMatrixFreeAccessToUninitializedMappingField(
5613 "update_hessians"));
5614 if constexpr (running_in_debug_mode())
5615 {
5616 this->hessians_quad_submitted = true;
5617 }
5618
5619 // compute hessian_unit = J^T * hessian_in(u) * J
5620 const std::size_t nqp = this->n_quadrature_points;
5621 constexpr unsigned int hdim = (dim * (dim + 1)) / 2;
5622 if (this->cell_type <= internal::MatrixFreeFunctions::affine)
5623 {
5624 const VectorizedArrayType JxW =
5625 this->J_value[0] * this->quadrature_weights[q_point];
5626
5627 const auto nxj = this->normal_x_jacobian[0];
5628
5629 // diagonal part
5630 for (unsigned int d = 0; d < dim; ++d)
5631 {
5632 const auto nxj_d = nxj[d];
5633 const VectorizedArrayType factor = nxj_d * nxj_d * JxW;
5634 for (unsigned int comp = 0; comp < n_components; ++comp)
5635 if constexpr (n_components == 1)
5636 this->hessians_quad[d * nqp + q_point] =
5637 normal_hessian_in * factor;
5638 else
5639 this->hessians_quad[(comp * hdim + d) * nqp + q_point] =
5640 normal_hessian_in[comp] * factor;
5641 }
5642
5643 // off diagonal part
5644 for (unsigned int d = 1, off_dia = dim; d < dim; ++d)
5645 for (unsigned int e = 0; e < d; ++e, ++off_dia)
5646 {
5647 const auto jac_d = nxj[d];
5648 const auto jac_e = nxj[e];
5649 const VectorizedArrayType factor = jac_d * jac_e * JxW;
5650 for (unsigned int comp = 0; comp < n_components; ++comp)
5651 if constexpr (n_components == 1)
5652 this->hessians_quad[off_dia * nqp + q_point] =
5653 2. * normal_hessian_in * factor;
5654 else
5655 this->hessians_quad[(comp * hdim + off_dia) * nqp + q_point] =
5656 2. * normal_hessian_in[comp] * factor;
5657 }
5658 }
5659 else
5660 {
5661 const auto normal = this->normal_vector(q_point);
5662 const auto normal_projector = outer_product(normal, normal);
5663 if constexpr (n_components == 1)
5664 submit_hessian(normal_hessian_in * normal_projector, q_point);
5665 else
5666 {
5667 hessian_type tmp;
5668 for (unsigned int comp = 0; comp < n_components; ++comp)
5669 tmp[comp] = normal_hessian_in[comp] * normal_projector;
5670 submit_hessian(tmp, q_point);
5671 }
5672 }
5673 }
5674
5675
5676
5677 template <int dim,
5678 int n_components_,
5679 typename Number,
5680 bool is_face,
5681 typename VectorizedArrayType>
5682 inline typename FEEvaluationBase<dim,
5683 n_components_,
5684 Number,
5685 is_face,
5686 VectorizedArrayType>::value_type
5687 FEEvaluationBase<dim, n_components_, Number, is_face, VectorizedArrayType>::
5688 integrate_value() const
5689 {
5690 if constexpr (running_in_debug_mode())
5691 {
5692 Assert(this->is_reinitialized, ExcNotInitialized());
5693 Assert(this->values_quad_submitted == true,
5694 internal::ExcAccessToUninitializedField());
5695 }
5696
5697 Tensor<1, n_components, VectorizedArrayType> return_value;
5698 const std::size_t nqp = this->n_quadrature_points;
5699 for (unsigned int q = 0; q < nqp; ++q)
5700 for (unsigned int comp = 0; comp < n_components; ++comp)
5701 return_value[comp] += this->values_quad[comp * nqp + q];
5702 if constexpr (n_components == 1)
5703 return return_value[0];
5704 else
5705 return return_value;
5706 }
5707
5708
5709
5710 template <int dim,
5711 int n_components_,
5712 typename Number,
5713 bool is_face,
5714 typename VectorizedArrayType>
5715 template <int, typename>
5716 inline DEAL_II_ALWAYS_INLINE VectorizedArrayType
5717 FEEvaluationBase<dim, n_components_, Number, is_face, VectorizedArrayType>::
5718 get_divergence(const unsigned int q_point) const
5719 {
5720 static_assert(n_components == dim,
5721 "Do not try to modify the default template parameters used for"
5722 " selectively enabling this function via std::enable_if!");
5723
5724 if constexpr (running_in_debug_mode())
5725 {
5726 Assert(this->gradients_quad_initialized == true,
5727 internal::ExcAccessToUninitializedField());
5728 }
5729 AssertIndexRange(q_point, this->n_quadrature_points);
5730 Assert(this->jacobian != nullptr,
5731 internal::ExcMatrixFreeAccessToUninitializedMappingField(
5732 "update_gradients"));
5733
5734 VectorizedArrayType divergence;
5735 const std::size_t nqp = this->n_quadrature_points;
5736
5737 if (dim > 1 &&
5738 this->data->element_type ==
5739 internal::MatrixFreeFunctions::ElementType::tensor_raviart_thomas)
5740 {
5741 VectorizedArrayType inv_det =
5742 (!is_face &&
5743 this->cell_type == internal::MatrixFreeFunctions::cartesian) ?
5744 this->jacobian[0][0][0] *
5745 ((dim == 2) ? this->jacobian[0][1][1] :
5746 this->jacobian[0][1][1] * this->jacobian[0][2][2]) :
5747 determinant(this->jacobian[this->cell_type >
5748 internal::MatrixFreeFunctions::affine ?
5749 q_point :
5750 0]);
5751
5752 // on faces in 2d, the determinant has the wrong sign due to ordering of
5753 // derivatives
5754 if (is_face && dim == 2 && this->get_face_no() < 2)
5755 inv_det = -inv_det;
5756
5757 // div * det(J^-1)
5758 divergence = this->gradients_quad[q_point * dim];
5759 for (unsigned int d = 1; d < dim; ++d)
5760 divergence += this->gradients_quad[(d * nqp + q_point) * dim + d];
5761 divergence *= inv_det;
5762 }
5763 else
5764 {
5765 if (!is_face &&
5766 this->cell_type == internal::MatrixFreeFunctions::cartesian)
5767 {
5768 // Cartesian cell
5769 divergence =
5770 this->gradients_quad[q_point * dim] * this->jacobian[0][0][0];
5771 for (unsigned int d = 1; d < dim; ++d)
5772 divergence += this->gradients_quad[(d * nqp + q_point) * dim + d] *
5773 this->jacobian[0][d][d];
5774 }
5775 else
5776 {
5777 // cell with general/constant Jacobian
5778 const Tensor<2, dim, VectorizedArrayType> &jac =
5779 this->cell_type == internal::MatrixFreeFunctions::general ?
5780 this->jacobian[q_point] :
5781 this->jacobian[0];
5782 divergence = jac[0][0] * this->gradients_quad[q_point * dim];
5783 for (unsigned int e = 1; e < dim; ++e)
5784 divergence += jac[0][e] * this->gradients_quad[q_point * dim + e];
5785 for (unsigned int d = 1; d < dim; ++d)
5786 for (unsigned int e = 0; e < dim; ++e)
5787 divergence +=
5788 jac[d][e] * this->gradients_quad[(d * nqp + q_point) * dim + e];
5789 }
5790 }
5791 return divergence;
5792 }
5793
5794
5795
5796 template <int dim,
5797 int n_components_,
5798 typename Number,
5799 bool is_face,
5800 typename VectorizedArrayType>
5801 template <int, typename>
5802 inline DEAL_II_ALWAYS_INLINE SymmetricTensor<2, dim, VectorizedArrayType>
5803 FEEvaluationBase<dim, n_components_, Number, is_face, VectorizedArrayType>::
5804 get_symmetric_gradient(const unsigned int q_point) const
5805 {
5806 static_assert(n_components == dim,
5807 "Do not try to modify the default template parameters used for"
5808 " selectively enabling this function via std::enable_if!");
5809
5810 // copy from generic function into dim-specialization function
5811 const auto grad = get_gradient(q_point);
5812 VectorizedArrayType symmetrized[(dim * dim + dim) / 2];
5813 VectorizedArrayType half = Number(0.5);
5814 for (unsigned int d = 0; d < dim; ++d)
5815 symmetrized[d] = grad[d][d];
5816 switch (dim)
5817 {
5818 case 1:
5819 break;
5820 case 2:
5821 symmetrized[2] = grad[0][1] + grad[1][0];
5822 symmetrized[2] *= half;
5823 break;
5824 case 3:
5825 symmetrized[3] = grad[0][1] + grad[1][0];
5826 symmetrized[3] *= half;
5827 symmetrized[4] = grad[0][2] + grad[2][0];
5828 symmetrized[4] *= half;
5829 symmetrized[5] = grad[1][2] + grad[2][1];
5830 symmetrized[5] *= half;
5831 break;
5832 default:
5833 DEAL_II_NOT_IMPLEMENTED();
5834 }
5835 return SymmetricTensor<2, dim, VectorizedArrayType>(symmetrized);
5836 }
5837
5838
5839
5840 template <int dim,
5841 int n_components_,
5842 typename Number,
5843 bool is_face,
5844 typename VectorizedArrayType>
5845 template <int, typename>
5846 inline DEAL_II_ALWAYS_INLINE
5847 Tensor<1, (dim == 2 ? 1 : dim), VectorizedArrayType>
5848 FEEvaluationBase<dim, n_components_, Number, is_face, VectorizedArrayType>::
5849 get_curl(const unsigned int q_point) const
5850 {
5851 static_assert(dim > 1 && n_components == dim,
5852 "Do not try to modify the default template parameters used for"
5853 " selectively enabling this function via std::enable_if!");
5854
5855 // copy from generic function into dim-specialization function
5856 const Tensor<2, dim, VectorizedArrayType> grad = get_gradient(q_point);
5857 Tensor<1, (dim == 2 ? 1 : dim), VectorizedArrayType> curl;
5858 switch (dim)
5859 {
5860 case 2:
5861 curl[0] = grad[1][0] - grad[0][1];
5862 break;
5863 case 3:
5864 curl[0] = grad[2][1] - grad[1][2];
5865 curl[1] = grad[0][2] - grad[2][0];
5866 curl[2] = grad[1][0] - grad[0][1];
5867 break;
5868 default:
5869 DEAL_II_NOT_IMPLEMENTED();
5870 }
5871 return curl;
5872 }
5873
5874
5875
5876 template <int dim,
5877 int n_components_,
5878 typename Number,
5879 bool is_face,
5880 typename VectorizedArrayType>
5881 template <int, typename>
5882 inline DEAL_II_ALWAYS_INLINE void
5883 FEEvaluationBase<dim, n_components_, Number, is_face, VectorizedArrayType>::
5884 submit_divergence(const VectorizedArrayType div_in,
5885 const unsigned int q_point)
5886 {
5887 static_assert(n_components == dim,
5888 "Do not try to modify the default template parameters used for"
5889 " selectively enabling this function via std::enable_if!");
5890
5891 if constexpr (running_in_debug_mode())
5892 {
5893 Assert(this->is_reinitialized, ExcNotInitialized());
5894 }
5895 AssertIndexRange(q_point, this->n_quadrature_points);
5896 Assert(this->J_value != nullptr,
5897 internal::ExcMatrixFreeAccessToUninitializedMappingField(
5898 "update_gradients"));
5899 Assert(this->jacobian != nullptr,
5900 internal::ExcMatrixFreeAccessToUninitializedMappingField(
5901 "update_gradients"));
5902 if constexpr (running_in_debug_mode())
5903 {
5904 this->gradients_quad_submitted = true;
5905 }
5906
5907 const std::size_t nqp_d = this->n_quadrature_points * dim;
5908 VectorizedArrayType *gradients = this->gradients_quad + q_point * dim;
5909
5910 if (this->data->element_type ==
5911 internal::MatrixFreeFunctions::ElementType::tensor_raviart_thomas)
5912 {
5913 // General cell
5914
5915 // Derivatives are reordered for faces. Need to take this into account
5916 // and 1/inv_det != J_value for faces
5917 const VectorizedArrayType fac =
5918 (!is_face) ?
5919 this->quadrature_weights[q_point] * div_in :
5920 (this->cell_type > internal::MatrixFreeFunctions::affine ?
5921 this->J_value[q_point] :
5922 this->J_value[0] * this->quadrature_weights[q_point]) *
5923 div_in *
5924 determinant(
5925 this->jacobian[this->cell_type >
5926 internal::MatrixFreeFunctions::affine ?
5927 q_point :
5928 0]) *
5929 Number((dim == 2 && this->get_face_no() < 2) ? -1 : 1);
5930
5931 for (unsigned int d = 0; d < dim; ++d)
5932 {
5933 for (unsigned int e = 0; e < dim; ++e)
5934 gradients[d * nqp_d + e] = (d == e) ? fac : 0.;
5935 }
5936 this->divergence_is_requested = true;
5937 }
5938 else
5939 {
5940 if (!is_face &&
5941 this->cell_type == internal::MatrixFreeFunctions::cartesian)
5942 {
5943 const VectorizedArrayType fac =
5944 this->J_value[0] * this->quadrature_weights[q_point] * div_in;
5945 for (unsigned int d = 0; d < dim; ++d)
5946 {
5947 const VectorizedArrayType jac_dd = this->jacobian[0][d][d];
5948 for (unsigned int e = 0; e < dim; ++e)
5949 gradients[d * nqp_d + e] = (d == e) ? fac * jac_dd : 0.;
5950 }
5951 }
5952 else
5953 {
5954 const Tensor<2, dim, VectorizedArrayType> jac =
5955 this->cell_type == internal::MatrixFreeFunctions::general ?
5956 this->jacobian[q_point] :
5957 this->jacobian[0];
5958 const VectorizedArrayType fac =
5959 (this->cell_type == internal::MatrixFreeFunctions::general ?
5960 this->J_value[q_point] :
5961 this->J_value[0] * this->quadrature_weights[q_point]) *
5962 div_in;
5963 for (unsigned int d = 0; d < dim; ++d)
5964 {
5965 for (unsigned int e = 0; e < dim; ++e)
5966 gradients[d * nqp_d + e] = jac[d][e] * fac;
5967 }
5968 }
5969 }
5970 }
5971
5972
5973
5974 template <int dim,
5975 int n_components_,
5976 typename Number,
5977 bool is_face,
5978 typename VectorizedArrayType>
5979 template <int, typename>
5980 inline DEAL_II_ALWAYS_INLINE void
5981 FEEvaluationBase<dim, n_components_, Number, is_face, VectorizedArrayType>::
5982 submit_symmetric_gradient(
5983 const SymmetricTensor<2, dim, VectorizedArrayType> sym_grad,
5984 const unsigned int q_point)
5985 {
5986 static_assert(n_components == dim,
5987 "Do not try to modify the default template parameters used for"
5988 " selectively enabling this function via std::enable_if!");
5989
5990 AssertThrow(
5991 this->data->element_type !=
5992 internal::MatrixFreeFunctions::ElementType::tensor_raviart_thomas,
5993 ExcNotImplemented());
5994
5995 // could have used base class operator, but that involves some overhead
5996 // which is inefficient. it is nice to have the symmetric tensor because
5997 // that saves some operations
5998 if constexpr (running_in_debug_mode())
5999 {
6000 Assert(this->is_reinitialized, ExcNotInitialized());
6001 }
6002 AssertIndexRange(q_point, this->n_quadrature_points);
6003 Assert(this->J_value != nullptr,
6004 internal::ExcMatrixFreeAccessToUninitializedMappingField(
6005 "update_gradients"));
6006 Assert(this->jacobian != nullptr,
6007 internal::ExcMatrixFreeAccessToUninitializedMappingField(
6008 "update_gradients"));
6009 if constexpr (running_in_debug_mode())
6010 {
6011 this->gradients_quad_submitted = true;
6012 }
6013
6014 const std::size_t nqp_d = this->n_quadrature_points * dim;
6015 VectorizedArrayType *gradients = this->gradients_quad + dim * q_point;
6016 if (!is_face && this->cell_type == internal::MatrixFreeFunctions::cartesian)
6017 {
6018 const VectorizedArrayType JxW =
6019 this->J_value[0] * this->quadrature_weights[q_point];
6020 const Tensor<2, dim, VectorizedArrayType> jac = this->jacobian[0];
6021 for (unsigned int d = 0; d < dim; ++d)
6022 gradients[d * nqp_d + d] =
6023 (sym_grad.access_raw_entry(d) * JxW * jac[d][d]);
6024 for (unsigned int e = 0, counter = dim; e < dim; ++e)
6025 for (unsigned int d = e + 1; d < dim; ++d, ++counter)
6026 {
6027 const VectorizedArrayType value =
6028 sym_grad.access_raw_entry(counter) * JxW;
6029 gradients[e * nqp_d + d] = value * jac[d][d];
6030 gradients[d * nqp_d + e] = value * jac[e][e];
6031 }
6032 }
6033 // general/affine cell type
6034 else
6035 {
6036 const VectorizedArrayType JxW =
6037 this->cell_type == internal::MatrixFreeFunctions::general ?
6038 this->J_value[q_point] :
6039 this->J_value[0] * this->quadrature_weights[q_point];
6040 const Tensor<2, dim, VectorizedArrayType> jac =
6041 this->cell_type == internal::MatrixFreeFunctions::general ?
6042 this->jacobian[q_point] :
6043 this->jacobian[0];
6044 VectorizedArrayType weighted[dim][dim];
6045 for (unsigned int i = 0; i < dim; ++i)
6046 weighted[i][i] = sym_grad.access_raw_entry(i) * JxW;
6047 for (unsigned int i = 0, counter = dim; i < dim; ++i)
6048 for (unsigned int j = i + 1; j < dim; ++j, ++counter)
6049 {
6050 const VectorizedArrayType value =
6051 sym_grad.access_raw_entry(counter) * JxW;
6052 weighted[i][j] = value;
6053 weighted[j][i] = value;
6054 }
6055 for (unsigned int comp = 0; comp < dim; ++comp)
6056 for (unsigned int d = 0; d < dim; ++d)
6057 {
6058 VectorizedArrayType new_val = jac[0][d] * weighted[comp][0];
6059 for (unsigned int e = 1; e < dim; ++e)
6060 new_val += jac[e][d] * weighted[comp][e];
6061 gradients[comp * nqp_d + d] = new_val;
6062 }
6063 }
6064 }
6065
6066
6067
6068 template <int dim,
6069 int n_components_,
6070 typename Number,
6071 bool is_face,
6072 typename VectorizedArrayType>
6073 template <int, typename>
6074 inline DEAL_II_ALWAYS_INLINE void
6075 FEEvaluationBase<dim, n_components_, Number, is_face, VectorizedArrayType>::
6076 submit_curl(const Tensor<1, dim == 2 ? 1 : dim, VectorizedArrayType> curl,
6077 const unsigned int q_point)
6078 {
6079 static_assert(n_components == dim,
6080 "Do not try to modify the default template parameters used for"
6081 " selectively enabling this function via std::enable_if!");
6082
6083 Tensor<2, dim, VectorizedArrayType> grad;
6084 switch (dim)
6085 {
6086 case 2:
6087 grad[1][0] = curl[0];
6088 grad[0][1] = -curl[0];
6089 break;
6090 case 3:
6091 grad[2][1] = curl[0];
6092 grad[1][2] = -curl[0];
6093 grad[0][2] = curl[1];
6094 grad[2][0] = -curl[1];
6095 grad[1][0] = curl[2];
6096 grad[0][1] = -curl[2];
6097 break;
6098 default:
6099 DEAL_II_NOT_IMPLEMENTED();
6100 }
6101 submit_gradient(grad, q_point);
6102 }
6103
6104
6105
6106 /*-------------------------- FEEvaluation -----------------------------------*/
6107
6108
6109 template <int dim,
6110 int fe_degree,
6111 int n_q_points_1d,
6112 int n_components_,
6113 typename Number,
6114 typename VectorizedArrayType>
6115 inline FEEvaluation<dim,
6116 fe_degree,
6117 n_q_points_1d,
6118 n_components_,
6119 Number,
6120 VectorizedArrayType>::
6121 FEEvaluation(const MatrixFree<dim, Number, VectorizedArrayType> &matrix_free,
6122 const unsigned int fe_no,
6123 const unsigned int quad_no,
6124 const unsigned int first_selected_component,
6125 const unsigned int active_fe_index,
6126 const unsigned int active_quad_index)
6127 : BaseClass(matrix_free,
6128 fe_no,
6129 first_selected_component,
6130 quad_no,
6131 fe_degree,
6132 static_n_q_points,
6133 true /*note: this is not a face*/,
6134 active_fe_index,
6135 active_quad_index,
6136 numbers::invalid_unsigned_int /*face_type*/)
6137 , dofs_per_component(this->data->dofs_per_component_on_cell)
6138 , dofs_per_cell(this->data->dofs_per_component_on_cell * n_components_)
6139 , n_q_points(this->data->n_q_points)
6140 {
6141 check_template_arguments(fe_no, 0);
6142 }
6143
6144
6145
6146 template <int dim,
6147 int fe_degree,
6148 int n_q_points_1d,
6149 int n_components_,
6150 typename Number,
6151 typename VectorizedArrayType>
6152 inline FEEvaluation<dim,
6153 fe_degree,
6154 n_q_points_1d,
6155 n_components_,
6156 Number,
6157 VectorizedArrayType>::
6158 FEEvaluation(const MatrixFree<dim, Number, VectorizedArrayType> &matrix_free,
6159 const std::pair<unsigned int, unsigned int> &range,
6160 const unsigned int dof_no,
6161 const unsigned int quad_no,
6162 const unsigned int first_selected_component)
6163 : FEEvaluation(matrix_free,
6164 dof_no,
6165 quad_no,
6166 first_selected_component,
6167 matrix_free.get_cell_active_fe_index(range, dof_no))
6168 {}
6169
6170
6171
6172 template <int dim,
6173 int fe_degree,
6174 int n_q_points_1d,
6175 int n_components_,
6176 typename Number,
6177 typename VectorizedArrayType>
6178 inline FEEvaluation<dim,
6179 fe_degree,
6180 n_q_points_1d,
6181 n_components_,
6182 Number,
6183 VectorizedArrayType>::
6184 FEEvaluation(const Mapping<dim> &mapping,
6185 const FiniteElement<dim> &fe,
6186 const Quadrature<1> &quadrature,
6187 const UpdateFlags update_flags,
6188 const unsigned int first_selected_component)
6189 : BaseClass(mapping,
6190 fe,
6191 quadrature,
6192 update_flags,
6193 first_selected_component,
6194 nullptr)
6195 , dofs_per_component(this->data->dofs_per_component_on_cell)
6196 , dofs_per_cell(this->data->dofs_per_component_on_cell * n_components_)
6197 , n_q_points(this->data->n_q_points)
6198 {
6199 check_template_arguments(numbers::invalid_unsigned_int, 0);
6200 }
6201
6202
6203
6204 template <int dim,
6205 int fe_degree,
6206 int n_q_points_1d,
6207 int n_components_,
6208 typename Number,
6209 typename VectorizedArrayType>
6210 inline FEEvaluation<dim,
6211 fe_degree,
6212 n_q_points_1d,
6213 n_components_,
6214 Number,
6215 VectorizedArrayType>::
6216 FEEvaluation(const FiniteElement<dim> &fe,
6217 const Quadrature<1> &quadrature,
6218 const UpdateFlags update_flags,
6219 const unsigned int first_selected_component)
6220 : BaseClass(StaticMappingQ1<dim>::mapping,
6221 fe,
6222 quadrature,
6223 update_flags,
6224 first_selected_component,
6225 nullptr)
6226 , dofs_per_component(this->data->dofs_per_component_on_cell)
6227 , dofs_per_cell(this->data->dofs_per_component_on_cell * n_components_)
6228 , n_q_points(this->data->n_q_points)
6229 {
6230 check_template_arguments(numbers::invalid_unsigned_int, 0);
6231 }
6232
6233
6234
6235 template <int dim,
6236 int fe_degree,
6237 int n_q_points_1d,
6238 int n_components_,
6239 typename Number,
6240 typename VectorizedArrayType>
6241 inline FEEvaluation<dim,
6242 fe_degree,
6243 n_q_points_1d,
6244 n_components_,
6245 Number,
6246 VectorizedArrayType>::
6247 FEEvaluation(const FiniteElement<dim> &fe,
6248 const FEEvaluationData<dim, VectorizedArrayType, false> &other,
6249 const unsigned int first_selected_component)
6250 : BaseClass(other.mapped_geometry->get_fe_values().get_mapping(),
6251 fe,
6252 other.mapped_geometry->get_quadrature(),
6253 other.mapped_geometry->get_fe_values().get_update_flags(),
6254 first_selected_component,
6255 &other)
6256 , dofs_per_component(this->data->dofs_per_component_on_cell)
6257 , dofs_per_cell(this->data->dofs_per_component_on_cell * n_components_)
6258 , n_q_points(this->data->n_q_points)
6259 {
6260 check_template_arguments(numbers::invalid_unsigned_int, 0);
6261 }
6262
6263
6264
6265 template <int dim,
6266 int fe_degree,
6267 int n_q_points_1d,
6268 int n_components_,
6269 typename Number,
6270 typename VectorizedArrayType>
6271 inline FEEvaluation<dim,
6272 fe_degree,
6273 n_q_points_1d,
6274 n_components_,
6275 Number,
6276 VectorizedArrayType>::FEEvaluation(const FEEvaluation
6277 &other)
6278 : BaseClass(other)
6279 , dofs_per_component(this->data->dofs_per_component_on_cell)
6280 , dofs_per_cell(this->data->dofs_per_component_on_cell * n_components_)
6281 , n_q_points(this->data->n_q_points)
6282 {
6283 check_template_arguments(numbers::invalid_unsigned_int, 0);
6284 }
6285
6286
6287
6288 template <int dim,
6289 int fe_degree,
6290 int n_q_points_1d,
6291 int n_components_,
6292 typename Number,
6293 typename VectorizedArrayType>
6294 inline FEEvaluation<dim,
6295 fe_degree,
6296 n_q_points_1d,
6297 n_components_,
6298 Number,
6299 VectorizedArrayType> &
6300 FEEvaluation<dim,
6301 fe_degree,
6302 n_q_points_1d,
6303 n_components_,
6304 Number,
6305 VectorizedArrayType>::operator=(const FEEvaluation &other)
6306 {
6307 BaseClass::operator=(other);
6308 check_template_arguments(numbers::invalid_unsigned_int, 0);
6309 return *this;
6310 }
6311
6312
6313
6314 template <int dim,
6315 int fe_degree,
6316 int n_q_points_1d,
6317 int n_components_,
6318 typename Number,
6319 typename VectorizedArrayType>
6320 inline void
6321 FEEvaluation<dim,
6322 fe_degree,
6323 n_q_points_1d,
6324 n_components_,
6325 Number,
6326 VectorizedArrayType>::
6327 check_template_arguments(const unsigned int dof_no,
6328 const unsigned int first_selected_component)
6329 {
6330 (void)dof_no;
6331 (void)first_selected_component;
6332
6333 Assert(
6334 this->data->dofs_per_component_on_cell > 0,
6335 ExcMessage(
6336 "There is nothing useful you can do with an FEEvaluation object with "
6337 "FE_Nothing, i.e., without DoFs! If you have passed to "
6338 "MatrixFree::reinit() a collection of finite elements also containing "
6339 "FE_Nothing, please check - before creating FEEvaluation - the category "
6340 "of the current range by calling either "
6341 "MatrixFree::get_cell_range_category(range) or "
6342 "MatrixFree::get_face_range_category(range). The returned category "
6343 "is the index of the active FE, which you can use to exclude "
6344 "FE_Nothing."));
6345
6346 if constexpr (running_in_debug_mode())
6347 {
6348 // print error message when the dimensions do not match. Propose a
6349 // possible fix
6350 if ((static_cast<unsigned int>(fe_degree) !=
6351 numbers::invalid_unsigned_int &&
6352 static_cast<unsigned int>(fe_degree) !=
6353 this->data->data.front().fe_degree) ||
6354 n_q_points != this->n_quadrature_points)
6355 {
6356 std::string message =
6357 "-------------------------------------------------------\n";
6358 message +=
6359 "Illegal arguments in constructor/wrong template arguments!\n";
6360 message += " Called --> FEEvaluation<dim,";
6361 message += Utilities::int_to_string(fe_degree) + ",";
6362 message += Utilities::int_to_string(n_q_points_1d);
6363 message += "," + Utilities::int_to_string(n_components);
6364 message += ",Number>(data";
6365 if (first_selected_component != numbers::invalid_unsigned_int)
6366 {
6367 message += ", " + Utilities::int_to_string(dof_no) + ", ";
6368 message += Utilities::int_to_string(this->quad_no) + ", ";
6369 message += Utilities::int_to_string(first_selected_component);
6370 }
6371 message += ")\n";
6372
6373 // check whether some other vector component has the correct number of
6374 // points
6375 unsigned int proposed_dof_comp = numbers::invalid_unsigned_int,
6376 proposed_fe_comp = numbers::invalid_unsigned_int,
6377 proposed_quad_comp = numbers::invalid_unsigned_int;
6378 if (dof_no != numbers::invalid_unsigned_int)
6379 {
6380 if (static_cast<unsigned int>(fe_degree) ==
6381 this->data->data.front().fe_degree)
6382 {
6383 proposed_dof_comp = dof_no;
6384 proposed_fe_comp = first_selected_component;
6385 }
6386 else
6387 for (unsigned int no = 0;
6388 no < this->matrix_free->n_components();
6389 ++no)
6390 for (unsigned int nf = 0;
6391 nf < this->matrix_free->n_base_elements(no);
6392 ++nf)
6393 if (this->matrix_free
6394 ->get_shape_info(no, 0, nf, this->active_fe_index, 0)
6395 .data.front()
6396 .fe_degree == static_cast<unsigned int>(fe_degree))
6397 {
6398 proposed_dof_comp = no;
6399 proposed_fe_comp = nf;
6400 break;
6401 }
6402 if (n_q_points ==
6403 this->mapping_data->descriptor[this->active_quad_index]
6404 .n_q_points)
6405 proposed_quad_comp = this->quad_no;
6406 else
6407 for (unsigned int no = 0;
6408 no <
6409 this->matrix_free->get_mapping_info().cell_data.size();
6410 ++no)
6411 if (this->matrix_free->get_mapping_info()
6412 .cell_data[no]
6413 .descriptor[this->active_quad_index]
6414 .n_q_points == n_q_points)
6415 {
6416 proposed_quad_comp = no;
6417 break;
6418 }
6419 }
6420 if (proposed_dof_comp != numbers::invalid_unsigned_int &&
6421 proposed_quad_comp != numbers::invalid_unsigned_int)
6422 {
6423 if (proposed_dof_comp != first_selected_component)
6424 message += "Wrong vector component selection:\n";
6425 else
6426 message += "Wrong quadrature formula selection:\n";
6427 message += " Did you mean FEEvaluation<dim,";
6428 message += Utilities::int_to_string(fe_degree) + ",";
6429 message += Utilities::int_to_string(n_q_points_1d);
6430 message += "," + Utilities::int_to_string(n_components);
6431 message += ",Number>(data";
6432 if (dof_no != numbers::invalid_unsigned_int)
6433 {
6434 message +=
6435 ", " + Utilities::int_to_string(proposed_dof_comp) + ", ";
6436 message +=
6437 Utilities::int_to_string(proposed_quad_comp) + ", ";
6438 message += Utilities::int_to_string(proposed_fe_comp);
6439 }
6440 message += ")?\n";
6441 std::string correct_pos;
6442 if (proposed_dof_comp != dof_no)
6443 correct_pos = " ^ ";
6444 else
6445 correct_pos = " ";
6446 if (proposed_quad_comp != this->quad_no)
6447 correct_pos += " ^ ";
6448 else
6449 correct_pos += " ";
6450 if (proposed_fe_comp != first_selected_component)
6451 correct_pos += " ^\n";
6452 else
6453 correct_pos += " \n";
6454 message +=
6455 " " +
6456 correct_pos;
6457 }
6458 // ok, did not find the numbers specified by the template arguments in
6459 // the given list. Suggest correct template arguments
6460 const unsigned int proposed_n_q_points_1d = static_cast<unsigned int>(
6461 std::pow(1.001 * this->n_quadrature_points, 1. / dim));
6462 message += "Wrong template arguments:\n";
6463 message += " Did you mean FEEvaluation<dim,";
6464 message +=
6465 Utilities::int_to_string(this->data->data.front().fe_degree) + ",";
6466 message += Utilities::int_to_string(proposed_n_q_points_1d);
6467 message += "," + Utilities::int_to_string(n_components);
6468 message += ",Number>(data";
6469 if (dof_no != numbers::invalid_unsigned_int)
6470 {
6471 message += ", " + Utilities::int_to_string(dof_no) + ", ";
6472 message += Utilities::int_to_string(this->quad_no);
6473 message +=
6474 ", " + Utilities::int_to_string(first_selected_component);
6475 }
6476 message += ")?\n";
6477 std::string correct_pos;
6478 if (this->data->data.front().fe_degree !=
6479 static_cast<unsigned int>(fe_degree))
6480 correct_pos = " ^";
6481 else
6482 correct_pos = " ";
6483 if (proposed_n_q_points_1d != n_q_points_1d)
6484 correct_pos += " ^\n";
6485 else
6486 correct_pos += " \n";
6487 message += " " + correct_pos;
6488
6489 Assert(static_cast<unsigned int>(fe_degree) ==
6490 this->data->data.front().fe_degree &&
6491 n_q_points == this->n_quadrature_points,
6492 ExcMessage(message));
6493 }
6494 if (dof_no != numbers::invalid_unsigned_int)
6495 AssertDimension(
6496 n_q_points,
6497 this->mapping_data->descriptor[this->active_quad_index].n_q_points);
6498 }
6499 }
6500
6501
6502
6503 template <int dim,
6504 int fe_degree,
6505 int n_q_points_1d,
6506 int n_components_,
6507 typename Number,
6508 typename VectorizedArrayType>
6509 inline void
6510 FEEvaluation<dim,
6511 fe_degree,
6512 n_q_points_1d,
6513 n_components_,
6514 Number,
6515 VectorizedArrayType>::reinit(const unsigned int cell_index)
6516 {
6517 Assert(this->matrix_free != nullptr,
6518 ExcMessage("FEEvaluation was initialized without a matrix-free object."
6519 " Integer indexing is not possible."));
6520
6521 Assert(this->dof_info != nullptr, ExcNotInitialized());
6522 Assert(this->mapping_data != nullptr, ExcNotInitialized());
6523 this->cell = cell_index;
6524 this->cell_type =
6525 this->matrix_free->get_mapping_info().get_cell_type(cell_index);
6526
6527 const unsigned int offsets =
6528 this->mapping_data->data_index_offsets[cell_index];
6529 this->jacobian = &this->mapping_data->jacobians[0][offsets];
6530 this->J_value = &this->mapping_data->JxW_values[offsets];
6531 if (!this->mapping_data->jacobian_gradients[0].empty())
6532 {
6533 this->jacobian_gradients =
6534 this->mapping_data->jacobian_gradients[0].data() + offsets;
6535 this->jacobian_gradients_non_inverse =
6536 this->mapping_data->jacobian_gradients_non_inverse[0].data() + offsets;
6537 }
6538
6539 if (this->matrix_free->n_active_entries_per_cell_batch(this->cell) == n_lanes)
6540 {
6541 DEAL_II_OPENMP_SIMD_PRAGMA
6542 for (unsigned int i = 0; i < n_lanes; ++i)
6543 this->cell_ids[i] = cell_index * n_lanes + i;
6544 }
6545 else
6546 {
6547 unsigned int i = 0;
6548 for (; i < this->matrix_free->n_active_entries_per_cell_batch(this->cell);
6549 ++i)
6550 this->cell_ids[i] = cell_index * n_lanes + i;
6551 for (; i < n_lanes; ++i)
6552 this->cell_ids[i] = numbers::invalid_unsigned_int;
6553 }
6554
6555 if (this->mapping_data->quadrature_points.empty() == false)
6556 this->quadrature_points =
6557 &this->mapping_data->quadrature_points
6558 [this->mapping_data->quadrature_point_offsets[this->cell]];
6559
6560 if constexpr (running_in_debug_mode())
6561 {
6562 this->is_reinitialized = true;
6563 this->dof_values_initialized = false;
6564 this->values_quad_initialized = false;
6565 this->gradients_quad_initialized = false;
6566 this->hessians_quad_initialized = false;
6567 }
6568 }
6569
6570
6571
6572 template <int dim,
6573 int fe_degree,
6574 int n_q_points_1d,
6575 int n_components_,
6576 typename Number,
6577 typename VectorizedArrayType>
6578 inline void
6579 FEEvaluation<dim,
6580 fe_degree,
6581 n_q_points_1d,
6582 n_components_,
6583 Number,
6584 VectorizedArrayType>::reinit(const std::array<unsigned int,
6585 n_lanes> &cell_ids)
6586 {
6587 Assert(this->dof_info != nullptr, ExcNotInitialized());
6588 Assert(this->mapping_data != nullptr, ExcNotInitialized());
6589
6590 this->cell = numbers::invalid_unsigned_int;
6591 this->cell_ids = cell_ids;
6592
6593 // determine type of cell batch
6594 this->cell_type = internal::MatrixFreeFunctions::GeometryType::cartesian;
6595
6596 for (unsigned int v = 0; v < n_lanes; ++v)
6597 {
6598 const unsigned int cell_index = cell_ids[v];
6599
6600 if (cell_index == numbers::invalid_unsigned_int)
6601 continue;
6602
6603 this->cell_type =
6604 std::max(this->cell_type,
6605 this->matrix_free->get_mapping_info().get_cell_type(
6606 cell_index / n_lanes));
6607 }
6608
6609 // allocate memory for internal data storage
6610 if (this->mapped_geometry == nullptr)
6611 this->mapped_geometry =
6612 std::make_shared<internal::MatrixFreeFunctions::
6613 MappingDataOnTheFly<dim, VectorizedArrayType>>();
6614
6615 auto &mapping_storage = this->mapped_geometry->get_data_storage();
6616
6617 auto &this_jacobian_data = mapping_storage.jacobians[0];
6618 auto &this_J_value_data = mapping_storage.JxW_values;
6619 auto &this_jacobian_gradients_data = mapping_storage.jacobian_gradients[0];
6620 auto &this_jacobian_gradients_non_inverse_data =
6621 mapping_storage.jacobian_gradients_non_inverse[0];
6622 auto &this_quadrature_points_data = mapping_storage.quadrature_points;
6623
6624 if (this->cell_type <= internal::MatrixFreeFunctions::GeometryType::affine)
6625 {
6626 if (this_jacobian_data.size() != 2)
6627 this_jacobian_data.resize_fast(2);
6628
6629 if (this_J_value_data.size() != 1)
6630 this_J_value_data.resize_fast(1);
6631
6632 const auto &update_flags_cells =
6633 this->matrix_free->get_mapping_info().update_flags_cells;
6634
6635 if (update_flags_cells & update_jacobian_grads &&
6636 this_jacobian_gradients_data.size() != 1)
6637 {
6638 this_jacobian_gradients_data.resize_fast(1);
6639 this_jacobian_gradients_non_inverse_data.resize_fast(1);
6640 }
6641
6642 if (update_flags_cells & update_quadrature_points &&
6643 this_quadrature_points_data.size() != 1)
6644 this_quadrature_points_data.resize_fast(1);
6645 }
6646 else
6647 {
6648 if (this_jacobian_data.size() != this->n_quadrature_points)
6649 this_jacobian_data.resize_fast(this->n_quadrature_points);
6650
6651 if (this_J_value_data.size() != this->n_quadrature_points)
6652 this_J_value_data.resize_fast(this->n_quadrature_points);
6653
6654 const auto &update_flags_cells =
6655 this->matrix_free->get_mapping_info().update_flags_cells;
6656
6657 if (update_flags_cells & update_jacobian_grads &&
6658 this_jacobian_gradients_data.size() != this->n_quadrature_points)
6659 {
6660 this_jacobian_gradients_data.resize_fast(this->n_quadrature_points);
6661 this_jacobian_gradients_non_inverse_data.resize_fast(
6662 this->n_quadrature_points);
6663 }
6664
6665 if (update_flags_cells & update_quadrature_points &&
6666 this_quadrature_points_data.size() != this->n_quadrature_points)
6667 this_quadrature_points_data.resize_fast(this->n_quadrature_points);
6668 }
6669
6670 // set pointers to internal data storage
6671 this->jacobian = this_jacobian_data.data();
6672 this->J_value = this_J_value_data.data();
6673 this->jacobian_gradients = this_jacobian_gradients_data.data();
6674 this->jacobian_gradients_non_inverse =
6675 this_jacobian_gradients_non_inverse_data.data();
6676 this->quadrature_points = this_quadrature_points_data.data();
6677
6678 // fill internal data storage lane by lane
6679 for (unsigned int v = 0; v < n_lanes; ++v)
6680 {
6681 const unsigned int cell_index = cell_ids[v];
6682
6683 if (cell_index == numbers::invalid_unsigned_int)
6684 continue;
6685
6686 const unsigned int cell_batch_index = cell_index / n_lanes;
6687 const unsigned int offsets =
6688 this->mapping_data->data_index_offsets[cell_batch_index];
6689 const unsigned int lane = cell_index % n_lanes;
6690
6691 if (this->cell_type <=
6692 internal::MatrixFreeFunctions::GeometryType::affine)
6693 {
6694 // case that all cells are Cartesian or affine
6695 for (unsigned int q = 0; q < 2; ++q)
6696 for (unsigned int i = 0; i < dim; ++i)
6697 for (unsigned int j = 0; j < dim; ++j)
6698 this_jacobian_data[q][i][j][v] =
6699 this->mapping_data->jacobians[0][offsets + q][i][j][lane];
6700
6701 const unsigned int q = 0;
6702
6703 this_J_value_data[q][v] =
6704 this->mapping_data->JxW_values[offsets + q][lane];
6705
6706 const auto &update_flags_cells =
6707 this->matrix_free->get_mapping_info().update_flags_cells;
6708
6709 if (update_flags_cells & update_jacobian_grads)
6710 {
6711 for (unsigned int i = 0; i < dim * (dim + 1) / 2; ++i)
6712 for (unsigned int j = 0; j < dim; ++j)
6713 this_jacobian_gradients_data[q][i][j][v] =
6714 this->mapping_data
6715 ->jacobian_gradients[0][offsets + q][i][j][lane];
6716
6717 for (unsigned int i = 0; i < dim * (dim + 1) / 2; ++i)
6718 for (unsigned int j = 0; j < dim; ++j)
6719 this_jacobian_gradients_non_inverse_data[q][i][j][v] =
6720 this->mapping_data
6721 ->jacobian_gradients_non_inverse[0][offsets + q][i][j]
6722 [lane];
6723 }
6724
6725 if (update_flags_cells & update_quadrature_points)
6726 for (unsigned int i = 0; i < dim; ++i)
6727 this_quadrature_points_data[q][i][v] =
6728 this->mapping_data->quadrature_points
6729 [this->mapping_data
6730 ->quadrature_point_offsets[cell_batch_index] +
6731 q][i][lane];
6732 }
6733 else
6734 {
6735 // general case that at least one cell is not Cartesian or affine
6736 const auto cell_type =
6737 this->matrix_free->get_mapping_info().get_cell_type(
6738 cell_batch_index);
6739
6740 for (unsigned int q = 0; q < this->n_quadrature_points; ++q)
6741 {
6742 const unsigned int q_src =
6743 (cell_type <=
6744 internal::MatrixFreeFunctions::GeometryType::affine) ?
6745 0 :
6746 q;
6747
6748 this_J_value_data[q][v] =
6749 this->mapping_data->JxW_values[offsets + q_src][lane];
6750
6751 for (unsigned int i = 0; i < dim; ++i)
6752 for (unsigned int j = 0; j < dim; ++j)
6753 this_jacobian_data[q][i][j][v] =
6754 this->mapping_data
6755 ->jacobians[0][offsets + q_src][i][j][lane];
6756
6757 const auto &update_flags_cells =
6758 this->matrix_free->get_mapping_info().update_flags_cells;
6759
6760 if (update_flags_cells & update_jacobian_grads)
6761 {
6762 for (unsigned int i = 0; i < dim * (dim + 1) / 2; ++i)
6763 for (unsigned int j = 0; j < dim; ++j)
6764 this_jacobian_gradients_data[q][i][j][v] =
6765 this->mapping_data
6766 ->jacobian_gradients[0][offsets + q_src][i][j][lane];
6767
6768 for (unsigned int i = 0; i < dim * (dim + 1) / 2; ++i)
6769 for (unsigned int j = 0; j < dim; ++j)
6770 this_jacobian_gradients_non_inverse_data[q][i][j][v] =
6771 this->mapping_data
6772 ->jacobian_gradients_non_inverse[0][offsets + q_src]
6773 [i][j][lane];
6774 }
6775
6776 if (update_flags_cells & update_quadrature_points)
6777 {
6778 if (cell_type <=
6779 internal::MatrixFreeFunctions::GeometryType::affine)
6780 {
6781 // affine case: quadrature points are not available but
6782 // have to be computed from the corner point and the
6783 // Jacobian
6784 Point<dim, VectorizedArrayType> point =
6785 this->mapping_data->quadrature_points
6786 [this->mapping_data
6787 ->quadrature_point_offsets[cell_batch_index] +
6788 0];
6789
6790 const Tensor<2, dim, VectorizedArrayType> &jac =
6791 this->mapping_data->jacobians[0][offsets + 1];
6792 if (cell_type == internal::MatrixFreeFunctions::cartesian)
6793 for (unsigned int d = 0; d < dim; ++d)
6794 point[d] +=
6795 jac[d][d] *
6796 static_cast<Number>(
6797 this->descriptor->quadrature.point(q)[d]);
6798 else
6799 for (unsigned int d = 0; d < dim; ++d)
6800 for (unsigned int e = 0; e < dim; ++e)
6801 point[d] +=
6802 jac[d][e] *
6803 static_cast<Number>(
6804 this->descriptor->quadrature.point(q)[e]);
6805
6806 for (unsigned int i = 0; i < dim; ++i)
6807 this_quadrature_points_data[q][i][v] = point[i][lane];
6808 }
6809 else
6810 {
6811 // general case: quadrature points are available
6812 for (unsigned int i = 0; i < dim; ++i)
6813 this_quadrature_points_data[q][i][v] =
6814 this->mapping_data->quadrature_points
6815 [this->mapping_data
6816 ->quadrature_point_offsets[cell_batch_index] +
6817 q][i][lane];
6818 }
6819 }
6820 }
6821 }
6822 }
6823
6824 if constexpr (running_in_debug_mode())
6825 {
6826 this->is_reinitialized = true;
6827 this->dof_values_initialized = false;
6828 this->values_quad_initialized = false;
6829 this->gradients_quad_initialized = false;
6830 this->hessians_quad_initialized = false;
6831 }
6832 }
6833
6834
6835
6836 template <int dim,
6837 int fe_degree,
6838 int n_q_points_1d,
6839 int n_components_,
6840 typename Number,
6841 typename VectorizedArrayType>
6842 template <bool level_dof_access>
6843 inline void
6844 FEEvaluation<dim,
6845 fe_degree,
6846 n_q_points_1d,
6847 n_components_,
6848 Number,
6849 VectorizedArrayType>::
6850 reinit(const TriaIterator<DoFCellAccessor<dim, dim, level_dof_access>> &cell)
6851 {
6852 Assert(this->matrix_free == nullptr,
6853 ExcMessage("Cannot use initialization from cell iterator if "
6854 "initialized from MatrixFree object. Use variant for "
6855 "on the fly computation with arguments as for FEValues "
6856 "instead"));
6857 Assert(this->mapped_geometry.get() != nullptr, ExcNotInitialized());
6858 this->mapped_geometry->reinit(
6859 static_cast<typename Triangulation<dim>::cell_iterator>(cell));
6860 this->local_dof_indices.resize(cell->get_fe().n_dofs_per_cell());
6861 if (level_dof_access)
6862 cell->get_mg_dof_indices(this->local_dof_indices);
6863 else
6864 cell->get_dof_indices(this->local_dof_indices);
6865
6866 if constexpr (running_in_debug_mode())
6867 {
6868 this->is_reinitialized = true;
6869 }
6870 }
6871
6872
6873
6874 template <int dim,
6875 int fe_degree,
6876 int n_q_points_1d,
6877 int n_components_,
6878 typename Number,
6879 typename VectorizedArrayType>
6880 inline void
6881 FEEvaluation<dim,
6882 fe_degree,
6883 n_q_points_1d,
6884 n_components_,
6885 Number,
6886 VectorizedArrayType>::
6887 reinit(const typename Triangulation<dim>::cell_iterator &cell)
6888 {
6889 Assert(this->matrix_free == 0,
6890 ExcMessage("Cannot use initialization from cell iterator if "
6891 "initialized from MatrixFree object. Use variant for "
6892 "on the fly computation with arguments as for FEValues "
6893 "instead"));
6894 Assert(this->mapped_geometry.get() != 0, ExcNotInitialized());
6895 this->mapped_geometry->reinit(cell);
6896
6897 if constexpr (running_in_debug_mode())
6898 {
6899 this->is_reinitialized = true;
6900 }
6901 }
6902
6903
6904
6905 template <int dim,
6906 int fe_degree,
6907 int n_q_points_1d,
6908 int n_components_,
6909 typename Number,
6910 typename VectorizedArrayType>
6911 inline void
6912 FEEvaluation<dim,
6913 fe_degree,
6914 n_q_points_1d,
6915 n_components_,
6916 Number,
6917 VectorizedArrayType>::
6918 evaluate(const EvaluationFlags::EvaluationFlags evaluation_flags)
6919 {
6920 if constexpr (running_in_debug_mode())
6921 {
6922 Assert(this->dof_values_initialized == true,
6923 internal::ExcAccessToUninitializedField());
6924 }
6925 evaluate(this->values_dofs, evaluation_flags);
6926 }
6927
6928
6929
6930 template <int dim,
6931 int fe_degree,
6932 int n_q_points_1d,
6933 int n_components_,
6934 typename Number,
6935 typename VectorizedArrayType>
6936 inline void
6937 FEEvaluation<dim,
6938 fe_degree,
6939 n_q_points_1d,
6940 n_components_,
6941 Number,
6942 VectorizedArrayType>::
6943 evaluate(const VectorizedArrayType *values_array,
6944 const EvaluationFlags::EvaluationFlags evaluation_flag)
6945 {
6946 const bool hessians_on_general_cells =
6947 evaluation_flag & EvaluationFlags::hessians &&
6948 (this->cell_type > internal::MatrixFreeFunctions::affine);
6949 EvaluationFlags::EvaluationFlags evaluation_flag_actual = evaluation_flag;
6950 if (hessians_on_general_cells)
6951 evaluation_flag_actual |= EvaluationFlags::gradients;
6952
6953 if (this->data->element_type ==
6954 internal::MatrixFreeFunctions::ElementType::tensor_raviart_thomas &&
6955 evaluation_flag & EvaluationFlags::gradients &&
6956 (this->cell_type > internal::MatrixFreeFunctions::affine))
6957 evaluation_flag_actual |= EvaluationFlags::values;
6958
6959 if constexpr (fe_degree > -1)
6960 {
6961 internal::FEEvaluationImplSelector<dim, VectorizedArrayType, false>::
6962 template run<fe_degree, n_q_points_1d>(n_components,
6963 evaluation_flag_actual,
6964 values_array,
6965 *this);
6966 }
6967 else
6968 {
6969 internal::FEEvaluationFactory<dim, VectorizedArrayType>::evaluate(
6970 n_components,
6971 evaluation_flag_actual,
6972 const_cast<VectorizedArrayType *>(values_array),
6973 *this);
6974 }
6975
6976 if constexpr (running_in_debug_mode())
6977 {
6978 this->values_quad_initialized =
6979 evaluation_flag_actual & EvaluationFlags::values;
6980 this->gradients_quad_initialized =
6981 evaluation_flag_actual & EvaluationFlags::gradients;
6982 this->hessians_quad_initialized =
6983 evaluation_flag_actual & EvaluationFlags::hessians;
6984 }
6985 }
6986
6987
6988 namespace internal
6989 {
6990 /**
6991 * Implementation for standard vectors (that have the begin() methods).
6992 */
6993 template <typename Number,
6994 typename VectorizedArrayType,
6995 typename VectorType,
6996 typename EvaluatorType,
6997 std::enable_if_t<internal::has_begin<VectorType> &&
6998 !IsBlockVector<VectorType>::value,
6999 VectorType> * = nullptr>
7000 VectorizedArrayType *
7001 check_vector_access_inplace(const EvaluatorType &fe_eval, VectorType &vector)
7002 {
7003 // for user-defined cell batches this functionality is not supported
7004 if (fe_eval.get_current_cell_index() == numbers::invalid_unsigned_int)
7005 return nullptr;
7006
7007 const unsigned int cell = fe_eval.get_cell_or_face_batch_id();
7008 const auto &dof_info = fe_eval.get_dof_info();
7009
7010 // If the index storage is interleaved and contiguous and the vector
7011 // storage has the correct alignment, we can directly pass the pointer
7012 // into the vector to the evaluate() and integrate() calls, without
7013 // reading the vector entries into a separate data field. This saves some
7014 // operations.
7015 if (std::is_same_v<typename VectorType::value_type, Number> &&
7016 dof_info.index_storage_variants
7017 [internal::MatrixFreeFunctions::DoFInfo::dof_access_cell][cell] ==
7018 internal::MatrixFreeFunctions::DoFInfo::IndexStorageVariants::
7019 interleaved_contiguous &&
7020 reinterpret_cast<std::size_t>(
7021 vector.begin() +
7022 dof_info.dof_indices_contiguous
7023 [internal::MatrixFreeFunctions::DoFInfo::dof_access_cell]
7024 [cell * VectorizedArrayType::size()]) %
7025 sizeof(VectorizedArrayType) ==
7026 0)
7027 {
7028 return reinterpret_cast<VectorizedArrayType *>(
7029 vector.begin() +
7030 dof_info.dof_indices_contiguous
7031 [internal::MatrixFreeFunctions::DoFInfo::dof_access_cell]
7032 [cell * VectorizedArrayType::size()] +
7033 dof_info.component_dof_indices_offset
7034 [fe_eval.get_active_fe_index()]
7035 [fe_eval.get_first_selected_component()] *
7036 VectorizedArrayType::size());
7037 }
7038 else
7039 return nullptr;
7040 }
7041
7042 /**
7043 * Implementation for block vectors.
7044 */
7045 template <typename Number,
7046 typename VectorizedArrayType,
7047 typename VectorType,
7048 typename EvaluatorType,
7049 std::enable_if_t<!internal::has_begin<VectorType> ||
7050 IsBlockVector<VectorType>::value,
7051 VectorType> * = nullptr>
7052 VectorizedArrayType *
7053 check_vector_access_inplace(const EvaluatorType &, VectorType &)
7054 {
7055 return nullptr;
7056 }
7057 } // namespace internal
7058
7059
7060
7061 template <int dim,
7062 int fe_degree,
7063 int n_q_points_1d,
7064 int n_components_,
7065 typename Number,
7066 typename VectorizedArrayType>
7067 template <typename VectorType>
7068 inline void
7069 FEEvaluation<dim,
7070 fe_degree,
7071 n_q_points_1d,
7072 n_components_,
7073 Number,
7074 VectorizedArrayType>::
7075 gather_evaluate(const VectorType &input_vector,
7076 const EvaluationFlags::EvaluationFlags evaluation_flag)
7077 {
7078 const VectorizedArrayType *src_ptr =
7079 internal::check_vector_access_inplace<Number, const VectorizedArrayType>(
7080 *this, input_vector);
7081 if (src_ptr != nullptr)
7082 evaluate(src_ptr, evaluation_flag);
7083 else
7084 {
7085 this->read_dof_values(input_vector);
7086 evaluate(this->begin_dof_values(), evaluation_flag);
7087 }
7088 }
7089
7090
7091
7092 template <int dim,
7093 int fe_degree,
7094 int n_q_points_1d,
7095 int n_components_,
7096 typename Number,
7097 typename VectorizedArrayType>
7098 inline void
7099 FEEvaluation<dim,
7100 fe_degree,
7101 n_q_points_1d,
7102 n_components_,
7103 Number,
7104 VectorizedArrayType>::
7105 integrate(const EvaluationFlags::EvaluationFlags integration_flag)
7106 {
7107 integrate(integration_flag, this->values_dofs);
7108
7109 if constexpr (running_in_debug_mode())
7110 {
7111 this->dof_values_initialized = true;
7112 }
7113 }
7114
7115
7116
7117 template <int dim,
7118 int fe_degree,
7119 int n_q_points_1d,
7120 int n_components_,
7121 typename Number,
7122 typename VectorizedArrayType>
7123 inline void
7124 FEEvaluation<dim,
7125 fe_degree,
7126 n_q_points_1d,
7127 n_components_,
7128 Number,
7129 VectorizedArrayType>::
7130 integrate(const EvaluationFlags::EvaluationFlags integration_flag,
7131 VectorizedArrayType *values_array,
7132 const bool sum_into_values_array)
7133 {
7134 if constexpr (running_in_debug_mode())
7135 {
7136 if (integration_flag & EvaluationFlags::values)
7137 Assert(this->values_quad_submitted == true,
7138 internal::ExcAccessToUninitializedField());
7139 if (integration_flag & EvaluationFlags::gradients)
7140 Assert(this->gradients_quad_submitted == true,
7141 internal::ExcAccessToUninitializedField());
7142 if ((integration_flag & EvaluationFlags::hessians) != 0u)
7143 Assert(this->hessians_quad_submitted == true,
7144 internal::ExcAccessToUninitializedField());
7145 }
7146 Assert(this->matrix_free != nullptr ||
7147 this->mapped_geometry->is_initialized(),
7148 ExcNotInitialized());
7149
7150 Assert(
7151 (integration_flag & ~(EvaluationFlags::values | EvaluationFlags::gradients |
7152 EvaluationFlags::hessians)) == 0,
7153 ExcMessage("Only EvaluationFlags::values, EvaluationFlags::gradients, and "
7154 "EvaluationFlags::hessians are supported."));
7155
7156 EvaluationFlags::EvaluationFlags integration_flag_actual = integration_flag;
7157 if (integration_flag & EvaluationFlags::hessians &&
7158 (this->cell_type > internal::MatrixFreeFunctions::affine))
7159 {
7160 unsigned int size = n_components * dim * n_q_points;
7161 if ((integration_flag & EvaluationFlags::gradients) != 0u)
7162 {
7163 for (unsigned int i = 0; i < size; ++i)
7164 this->gradients_quad[i] += this->gradients_from_hessians_quad[i];
7165 }
7166 else
7167 {
7168 for (unsigned int i = 0; i < size; ++i)
7169 this->gradients_quad[i] = this->gradients_from_hessians_quad[i];
7170 integration_flag_actual |= EvaluationFlags::gradients;
7171 }
7172 }
7173
7174 if (n_components == dim &&
7175 this->data->element_type ==
7176 internal::MatrixFreeFunctions::ElementType::tensor_raviart_thomas &&
7177 integration_flag & EvaluationFlags::gradients &&
7178 this->cell_type > internal::MatrixFreeFunctions::affine &&
7179 this->divergence_is_requested == false)
7180 {
7181 unsigned int size = n_components * n_q_points;
7182 if ((integration_flag & EvaluationFlags::values) != 0u)
7183 {
7184 for (unsigned int i = 0; i < size; ++i)
7185 this->values_quad[i] += this->values_from_gradients_quad[i];
7186 }
7187 else
7188 {
7189 for (unsigned int i = 0; i < size; ++i)
7190 this->values_quad[i] = this->values_from_gradients_quad[i];
7191 integration_flag_actual |= EvaluationFlags::values;
7192 }
7193 }
7194
7195 if constexpr (fe_degree > -1)
7196 {
7197 internal::FEEvaluationImplSelector<dim, VectorizedArrayType, true>::
7198 template run<fe_degree, n_q_points_1d>(n_components,
7199 integration_flag_actual,
7200 values_array,
7201 *this,
7202 sum_into_values_array);
7203 }
7204 else
7205 {
7206 internal::FEEvaluationFactory<dim, VectorizedArrayType>::integrate(
7207 n_components,
7208 integration_flag_actual,
7209 values_array,
7210 *this,
7211 sum_into_values_array);
7212 }
7213
7214 if constexpr (running_in_debug_mode())
7215 {
7216 this->dof_values_initialized = true;
7217 }
7218 }
7219
7220
7221
7222 template <int dim,
7223 int fe_degree,
7224 int n_q_points_1d,
7225 int n_components_,
7226 typename Number,
7227 typename VectorizedArrayType>
7228 template <typename VectorType>
7229 inline void
7230 FEEvaluation<dim,
7231 fe_degree,
7232 n_q_points_1d,
7233 n_components_,
7234 Number,
7235 VectorizedArrayType>::
7236 integrate_scatter(const EvaluationFlags::EvaluationFlags integration_flag,
7237 VectorType &destination)
7238 {
7239 VectorizedArrayType *dst_ptr =
7240 internal::check_vector_access_inplace<Number, VectorizedArrayType>(
7241 *this, destination);
7242 if (dst_ptr != nullptr)
7243 integrate(integration_flag, dst_ptr, true);
7244 else
7245 {
7246 integrate(integration_flag, this->begin_dof_values());
7247 this->distribute_local_to_global(destination);
7248 }
7249 }
7250
7251
7252
7253 template <int dim,
7254 int fe_degree,
7255 int n_q_points_1d,
7256 int n_components_,
7257 typename Number,
7258 typename VectorizedArrayType>
7259 inline std_cxx20::ranges::iota_view<unsigned int, unsigned int>
7260 FEEvaluation<dim,
7261 fe_degree,
7262 n_q_points_1d,
7263 n_components_,
7264 Number,
7265 VectorizedArrayType>::dof_indices() const
7266 {
7267 return std_cxx20::ranges::iota_view<unsigned int, unsigned int>(
7268 0U, dofs_per_cell);
7269 }
7270
7271
7272
7273 /*-------------------------- FEFaceEvaluation ---------------------------*/
7274
7275
7276
7277 template <int dim,
7278 int fe_degree,
7279 int n_q_points_1d,
7280 int n_components_,
7281 typename Number,
7282 typename VectorizedArrayType>
7283 inline FEFaceEvaluation<dim,
7284 fe_degree,
7285 n_q_points_1d,
7286 n_components_,
7287 Number,
7288 VectorizedArrayType>::
7289 FEFaceEvaluation(
7290 const MatrixFree<dim, Number, VectorizedArrayType> &matrix_free,
7291 const bool is_interior_face,
7292 const unsigned int dof_no,
7293 const unsigned int quad_no,
7294 const unsigned int first_selected_component,
7295 const unsigned int active_fe_index,
7296 const unsigned int active_quad_index,
7297 const unsigned int face_type)
7298 : BaseClass(matrix_free,
7299 dof_no,
7300 first_selected_component,
7301 quad_no,
7302 fe_degree,
7303 static_n_q_points,
7304 is_interior_face,
7305 active_fe_index,
7306 active_quad_index,
7307 face_type)
7308 , dofs_per_component(this->data->dofs_per_component_on_cell)
7309 , dofs_per_cell(this->data->dofs_per_component_on_cell * n_components_)
7310 , n_q_points(this->n_quadrature_points)
7311 {}
7312
7313
7314
7315 template <int dim,
7316 int fe_degree,
7317 int n_q_points_1d,
7318 int n_components_,
7319 typename Number,
7320 typename VectorizedArrayType>
7321 inline FEFaceEvaluation<dim,
7322 fe_degree,
7323 n_q_points_1d,
7324 n_components_,
7325 Number,
7326 VectorizedArrayType>::
7327 FEFaceEvaluation(
7328 const MatrixFree<dim, Number, VectorizedArrayType> &matrix_free,
7329 const std::pair<unsigned int, unsigned int> &range,
7330 const bool is_interior_face,
7331 const unsigned int dof_no,
7332 const unsigned int quad_no,
7333 const unsigned int first_selected_component)
7334 : FEFaceEvaluation(matrix_free,
7335 is_interior_face,
7336 dof_no,
7337 quad_no,
7338 first_selected_component,
7339 matrix_free.get_face_active_fe_index(range,
7340 is_interior_face),
7341 numbers::invalid_unsigned_int,
7342 matrix_free.get_face_info(range.first).face_type)
7343 {}
7344
7345
7346
7347 template <int dim,
7348 int fe_degree,
7349 int n_q_points_1d,
7350 int n_components_,
7351 typename Number,
7352 typename VectorizedArrayType>
7353 inline void
7354 FEFaceEvaluation<dim,
7355 fe_degree,
7356 n_q_points_1d,
7357 n_components_,
7358 Number,
7359 VectorizedArrayType>::reinit(const unsigned int face_index)
7360 {
7361 Assert(this->mapped_geometry == nullptr,
7362 ExcMessage("FEEvaluation was initialized without a matrix-free object."
7363 " Integer indexing is not possible"));
7364 if (this->mapped_geometry != nullptr)
7365 return;
7366
7367 this->cell = face_index;
7368 this->dof_access_index =
7369 this->is_interior_face() ?
7370 internal::MatrixFreeFunctions::DoFInfo::dof_access_face_interior :
7371 internal::MatrixFreeFunctions::DoFInfo::dof_access_face_exterior;
7372 Assert(this->mapping_data != nullptr, ExcNotInitialized());
7373
7374 if (face_index >=
7375 this->matrix_free->get_task_info().face_partition_data.back() &&
7376 face_index <
7377 this->matrix_free->get_task_info().boundary_partition_data.back())
7378 Assert(this->is_interior_face(),
7379 ExcMessage(
7380 "Boundary faces do not have a neighbor. When looping over "
7381 "boundary faces use FEFaceEvaluation with the parameter "
7382 "is_interior_face set to true. "));
7383
7384 this->reinit_face(this->matrix_free->get_face_info(face_index));
7385
7386 unsigned int i = 0;
7387 for (; i < this->matrix_free->n_active_entries_per_face_batch(this->cell);
7388 ++i)
7389 this->face_ids[i] = face_index * n_lanes + i;
7390 for (; i < n_lanes; ++i)
7391 this->face_ids[i] = numbers::invalid_unsigned_int;
7392
7393 this->cell_type = this->matrix_free->get_mapping_info().face_type[face_index];
7394 const unsigned int offsets =
7395 this->mapping_data->data_index_offsets[face_index];
7396 this->J_value = &this->mapping_data->JxW_values[offsets];
7397 this->normal_vectors = &this->mapping_data->normal_vectors[offsets];
7398 this->jacobian =
7399 &this->mapping_data->jacobians[!this->is_interior_face()][offsets];
7400 this->normal_x_jacobian =
7401 &this->mapping_data
7402 ->normals_times_jacobians[!this->is_interior_face()][offsets];
7403 this->jacobian_gradients =
7404 this->mapping_data->jacobian_gradients[!this->is_interior_face()].data() +
7405 offsets;
7406 this->jacobian_gradients_non_inverse =
7407 this->mapping_data
7408 ->jacobian_gradients_non_inverse[!this->is_interior_face()]
7409 .data() +
7410 offsets;
7411
7412 if (this->mapping_data->quadrature_point_offsets.empty() == false)
7413 {
7414 AssertIndexRange(this->cell,
7415 this->mapping_data->quadrature_point_offsets.size());
7416 this->quadrature_points =
7417 this->mapping_data->quadrature_points.data() +
7418 this->mapping_data->quadrature_point_offsets[this->cell];
7419 }
7420
7421 if constexpr (running_in_debug_mode())
7422 {
7423 this->is_reinitialized = true;
7424 this->dof_values_initialized = false;
7425 this->values_quad_initialized = false;
7426 this->gradients_quad_initialized = false;
7427 this->hessians_quad_initialized = false;
7428 }
7429 }
7430
7431
7432
7433 template <int dim,
7434 int fe_degree,
7435 int n_q_points_1d,
7436 int n_components_,
7437 typename Number,
7438 typename VectorizedArrayType>
7439 inline void
7440 FEFaceEvaluation<dim,
7441 fe_degree,
7442 n_q_points_1d,
7443 n_components_,
7444 Number,
7445 VectorizedArrayType>::reinit(const unsigned int cell_index,
7446 const unsigned int face_number)
7447 {
7448 Assert(
7449 this->quad_no <
7450 this->matrix_free->get_mapping_info().face_data_by_cells.size(),
7451 ExcMessage(
7452 "You must set MatrixFree::AdditionalData::mapping_update_flags_faces_by_cells to use the present reinit method."));
7453 AssertIndexRange(face_number, GeometryInfo<dim>::faces_per_cell);
7454 AssertIndexRange(cell_index,
7455 this->matrix_free->get_mapping_info().cell_type.size());
7456 Assert(this->mapped_geometry == nullptr,
7457 ExcMessage("FEEvaluation was initialized without a matrix-free object."
7458 " Integer indexing is not possible"));
7459 if (this->mapped_geometry != nullptr)
7460 return;
7461 Assert(this->matrix_free != nullptr, ExcNotInitialized());
7462
7463 this->cell_type = this->matrix_free->get_mapping_info()
7464 .faces_by_cells_type[cell_index][face_number];
7465 this->cell = cell_index;
7466 this->subface_index = GeometryInfo<dim>::max_children_per_cell;
7467 this->dof_access_index =
7468 internal::MatrixFreeFunctions::DoFInfo::dof_access_cell;
7469
7470 if (this->is_interior_face() == false)
7471 {
7472 // for this case, we need to look into the FaceInfo field that collects
7473 // information from both sides of a face once for the global mesh, and
7474 // pick the face id that is not the local one (cell_this).
7475 for (unsigned int i = 0; i < n_lanes; ++i)
7476 {
7477 // compute actual (non vectorized) cell ID
7478 const unsigned int cell_this = cell_index * n_lanes + i;
7479 // compute face ID
7480 unsigned int face_index =
7481 this->matrix_free->get_cell_and_face_to_plain_faces()(cell_index,
7482 face_number,
7483 i);
7484
7485 this->face_ids[i] = face_index;
7486
7487 if (face_index == numbers::invalid_unsigned_int)
7488 {
7489 this->cell_ids[i] = numbers::invalid_unsigned_int;
7490 this->face_numbers[i] = static_cast<std::uint8_t>(-1);
7491 this->face_orientations[i] = static_cast<std::uint8_t>(-1);
7492 continue; // invalid face ID: no neighbor on boundary
7493 }
7494
7495 const auto &faces =
7496 this->matrix_free->get_face_info(face_index / n_lanes);
7497 // get cell ID on both sides of face
7498 auto cell_m = faces.cells_interior[face_index % n_lanes];
7499 auto cell_p = faces.cells_exterior[face_index % n_lanes];
7500
7501 const bool face_identifies_as_interior = cell_m != cell_this;
7502
7503 Assert(cell_m == cell_this || cell_p == cell_this,
7504 ExcInternalError());
7505
7506 // compare the IDs with the given cell ID
7507 if (face_identifies_as_interior)
7508 {
7509 this->cell_ids[i] = cell_m; // neighbor has the other ID
7510 this->face_numbers[i] = faces.interior_face_no;
7511 }
7512 else
7513 {
7514 this->cell_ids[i] = cell_p;
7515 this->face_numbers[i] = faces.exterior_face_no;
7516 }
7517
7518 const bool orientation_interior_face = faces.face_orientation >= 8;
7519 auto face_orientation = faces.face_orientation % 8;
7520 if (face_identifies_as_interior != orientation_interior_face)
7521 {
7522 Assert(this->matrix_free->get_cell_iterator(cell_index, i)
7523 ->reference_cell() ==
7524 ReferenceCells::get_hypercube<dim>(),
7525 ExcNotImplemented());
7526 face_orientation =
7527 ReferenceCells::get_hypercube<dim - 1>()
7528 .get_inverse_combined_orientation(face_orientation);
7529 }
7530 this->face_orientations[i] = face_orientation;
7531 }
7532 }
7533 else
7534 {
7535 this->face_orientations[0] = 0;
7536 this->face_numbers[0] = face_number;
7537 if (this->matrix_free->n_active_entries_per_cell_batch(this->cell) ==
7538 n_lanes)
7539 {
7540 DEAL_II_OPENMP_SIMD_PRAGMA
7541 for (unsigned int i = 0; i < n_lanes; ++i)
7542 this->cell_ids[i] = cell_index * n_lanes + i;
7543 }
7544 else
7545 {
7546 unsigned int i = 0;
7547 for (; i <
7548 this->matrix_free->n_active_entries_per_cell_batch(this->cell);
7549 ++i)
7550 this->cell_ids[i] = cell_index * n_lanes + i;
7551 for (; i < n_lanes; ++i)
7552 this->cell_ids[i] = numbers::invalid_unsigned_int;
7553 }
7554 for (unsigned int i = 0; i < n_lanes; ++i)
7555 this->face_ids[i] =
7556 this->matrix_free->get_cell_and_face_to_plain_faces()(cell_index,
7557 face_number,
7558 i);
7559 }
7560
7561 const unsigned int offsets =
7562 this->matrix_free->get_mapping_info()
7563 .face_data_by_cells[this->quad_no]
7564 .data_index_offsets[cell_index * GeometryInfo<dim>::faces_per_cell +
7565 face_number];
7566 AssertIndexRange(offsets,
7567 this->matrix_free->get_mapping_info()
7568 .face_data_by_cells[this->quad_no]
7569 .JxW_values.size());
7570 this->J_value = &this->matrix_free->get_mapping_info()
7571 .face_data_by_cells[this->quad_no]
7572 .JxW_values[offsets];
7573 this->normal_vectors = &this->matrix_free->get_mapping_info()
7574 .face_data_by_cells[this->quad_no]
7575 .normal_vectors[offsets];
7576 this->jacobian = &this->matrix_free->get_mapping_info()
7577 .face_data_by_cells[this->quad_no]
7578 .jacobians[!this->is_interior_face()][offsets];
7579 this->normal_x_jacobian =
7580 &this->matrix_free->get_mapping_info()
7581 .face_data_by_cells[this->quad_no]
7582 .normals_times_jacobians[!this->is_interior_face()][offsets];
7583 this->jacobian_gradients =
7584 this->mapping_data->jacobian_gradients[!this->is_interior_face()].data() +
7585 offsets;
7586 this->jacobian_gradients_non_inverse =
7587 this->mapping_data
7588 ->jacobian_gradients_non_inverse[!this->is_interior_face()]
7589 .data() +
7590 offsets;
7591
7592 if (this->matrix_free->get_mapping_info()
7593 .face_data_by_cells[this->quad_no]
7594 .quadrature_point_offsets.empty() == false)
7595 {
7596 const unsigned int index =
7597 this->cell * GeometryInfo<dim>::faces_per_cell + this->face_numbers[0];
7598 AssertIndexRange(index,
7599 this->matrix_free->get_mapping_info()
7600 .face_data_by_cells[this->quad_no]
7601 .quadrature_point_offsets.size());
7602 this->quadrature_points = this->matrix_free->get_mapping_info()
7603 .face_data_by_cells[this->quad_no]
7604 .quadrature_points.data() +
7605 this->matrix_free->get_mapping_info()
7606 .face_data_by_cells[this->quad_no]
7607 .quadrature_point_offsets[index];
7608 }
7609
7610 if constexpr (running_in_debug_mode())
7611 {
7612 this->is_reinitialized = true;
7613 this->dof_values_initialized = false;
7614 this->values_quad_initialized = false;
7615 this->gradients_quad_initialized = false;
7616 this->hessians_quad_initialized = false;
7617 }
7618 }
7619
7620
7621
7622 template <int dim,
7623 int fe_degree,
7624 int n_q_points_1d,
7625 int n_components_,
7626 typename Number,
7627 typename VectorizedArrayType>
7628 inline void
7629 FEFaceEvaluation<dim,
7630 fe_degree,
7631 n_q_points_1d,
7632 n_components_,
7633 Number,
7634 VectorizedArrayType>::
7635 evaluate(const EvaluationFlags::EvaluationFlags evaluation_flag)
7636 {
7637 if constexpr (running_in_debug_mode())
7638 {
7639 Assert(this->dof_values_initialized, ExcNotInitialized());
7640 }
7641
7642 evaluate(this->values_dofs, evaluation_flag);
7643 }
7644
7645
7646
7647 template <int dim,
7648 int fe_degree,
7649 int n_q_points_1d,
7650 int n_components_,
7651 typename Number,
7652 typename VectorizedArrayType>
7653 inline void
7654 FEFaceEvaluation<dim,
7655 fe_degree,
7656 n_q_points_1d,
7657 n_components_,
7658 Number,
7659 VectorizedArrayType>::
7660 evaluate(const VectorizedArrayType *values_array,
7661 const EvaluationFlags::EvaluationFlags evaluation_flag)
7662 {
7663 Assert((evaluation_flag &
7664 ~(EvaluationFlags::values | EvaluationFlags::gradients |
7665 EvaluationFlags::hessians)) == 0,
7666 ExcMessage("Only EvaluationFlags::values, EvaluationFlags::gradients, "
7667 "and EvaluationFlags::hessians are supported."));
7668
7669 const bool hessians_on_general_cells =
7670 evaluation_flag & EvaluationFlags::hessians &&
7671 (this->cell_type > internal::MatrixFreeFunctions::affine);
7672 EvaluationFlags::EvaluationFlags evaluation_flag_actual = evaluation_flag;
7673 if (hessians_on_general_cells)
7674 evaluation_flag_actual |= EvaluationFlags::gradients;
7675
7676 if (this->data->element_type ==
7677 internal::MatrixFreeFunctions::ElementType::tensor_raviart_thomas &&
7678 evaluation_flag & EvaluationFlags::gradients &&
7679 (this->cell_type > internal::MatrixFreeFunctions::affine))
7680 evaluation_flag_actual |= EvaluationFlags::values;
7681
7682 if constexpr (fe_degree > -1)
7683 internal::FEFaceEvaluationImplEvaluateSelector<dim, VectorizedArrayType>::
7684 template run<fe_degree, n_q_points_1d>(n_components,
7685 evaluation_flag_actual,
7686 values_array,
7687 *this);
7688 else
7689 internal::FEFaceEvaluationFactory<dim, VectorizedArrayType>::evaluate(
7690 n_components, evaluation_flag_actual, values_array, *this);
7691
7692 if constexpr (running_in_debug_mode())
7693 {
7694 this->values_quad_initialized =
7695 evaluation_flag_actual & EvaluationFlags::values;
7696 this->gradients_quad_initialized =
7697 evaluation_flag_actual & EvaluationFlags::gradients;
7698 this->hessians_quad_initialized =
7699 evaluation_flag_actual & EvaluationFlags::hessians;
7700 }
7701 }
7702
7703
7704
7705 template <int dim,
7706 int fe_degree,
7707 int n_q_points_1d,
7708 int n_components_,
7709 typename Number,
7710 typename VectorizedArrayType>
7711 inline void
7712 FEFaceEvaluation<dim,
7713 fe_degree,
7714 n_q_points_1d,
7715 n_components_,
7716 Number,
7717 VectorizedArrayType>::
7718 project_to_face(const EvaluationFlags::EvaluationFlags evaluation_flag)
7719 {
7720 if constexpr (running_in_debug_mode())
7721 {
7722 Assert(this->dof_values_initialized, ExcNotInitialized());
7723 }
7724
7725 project_to_face(this->values_dofs, evaluation_flag);
7726 }
7727
7728
7729
7730 template <int dim,
7731 int fe_degree,
7732 int n_q_points_1d,
7733 int n_components_,
7734 typename Number,
7735 typename VectorizedArrayType>
7736 inline void
7737 FEFaceEvaluation<dim,
7738 fe_degree,
7739 n_q_points_1d,
7740 n_components_,
7741 Number,
7742 VectorizedArrayType>::
7743 project_to_face(const VectorizedArrayType *values_array,
7744 const EvaluationFlags::EvaluationFlags evaluation_flag)
7745 {
7746 Assert((evaluation_flag &
7747 ~(EvaluationFlags::values | EvaluationFlags::gradients |
7748 EvaluationFlags::hessians)) == 0,
7749 ExcMessage("Only EvaluationFlags::values, EvaluationFlags::gradients, "
7750 "and EvaluationFlags::hessians are supported."));
7751
7752 const bool hessians_on_general_cells =
7753 evaluation_flag & EvaluationFlags::hessians &&
7754 (this->cell_type > internal::MatrixFreeFunctions::affine);
7755 EvaluationFlags::EvaluationFlags evaluation_flag_actual = evaluation_flag;
7756 if (hessians_on_general_cells)
7757 evaluation_flag_actual |= EvaluationFlags::gradients;
7758
7759 if (this->data->element_type ==
7760 internal::MatrixFreeFunctions::ElementType::tensor_raviart_thomas &&
7761 evaluation_flag & EvaluationFlags::gradients &&
7762 (this->cell_type > internal::MatrixFreeFunctions::affine))
7763 evaluation_flag_actual |= EvaluationFlags::values;
7764
7765 if constexpr (fe_degree > -1)
7766 internal::FEFaceEvaluationImplProjectToFaceSelector<
7767 dim,
7768 VectorizedArrayType>::template run<fe_degree>(n_components,
7769 evaluation_flag_actual,
7770 values_array,
7771 *this);
7772 else
7773 internal::FEFaceEvaluationFactory<dim, VectorizedArrayType>::
7774 project_to_face(n_components,
7775 evaluation_flag_actual,
7776 values_array,
7777 *this);
7778
7779 // face dofs initialized
7780 }
7781
7782
7783
7784 template <int dim,
7785 int fe_degree,
7786 int n_q_points_1d,
7787 int n_components_,
7788 typename Number,
7789 typename VectorizedArrayType>
7790 inline void
7791 FEFaceEvaluation<dim,
7792 fe_degree,
7793 n_q_points_1d,
7794 n_components_,
7795 Number,
7796 VectorizedArrayType>::
7797 evaluate_in_face(const EvaluationFlags::EvaluationFlags evaluation_flag)
7798 {
7799 Assert((evaluation_flag &
7800 ~(EvaluationFlags::values | EvaluationFlags::gradients |
7801 EvaluationFlags::hessians)) == 0,
7802 ExcMessage("Only EvaluationFlags::values, EvaluationFlags::gradients, "
7803 "and EvaluationFlags::hessians are supported."));
7804
7805 const bool hessians_on_general_cells =
7806 evaluation_flag & EvaluationFlags::hessians &&
7807 (this->cell_type > internal::MatrixFreeFunctions::affine);
7808 EvaluationFlags::EvaluationFlags evaluation_flag_actual = evaluation_flag;
7809 if (hessians_on_general_cells)
7810 evaluation_flag_actual |= EvaluationFlags::gradients;
7811
7812 if (this->data->element_type ==
7813 internal::MatrixFreeFunctions::ElementType::tensor_raviart_thomas &&
7814 evaluation_flag & EvaluationFlags::gradients &&
7815 (this->cell_type > internal::MatrixFreeFunctions::affine))
7816 evaluation_flag_actual |= EvaluationFlags::values;
7817
7818 if constexpr (fe_degree > -1)
7819 internal::FEFaceEvaluationImplEvaluateInFaceSelector<
7820 dim,
7821 VectorizedArrayType>::template run<fe_degree>(n_components,
7822 evaluation_flag_actual,
7823 *this);
7824 else
7825 internal::FEFaceEvaluationFactory<dim, VectorizedArrayType>::
7826 evaluate_in_face(n_components, evaluation_flag_actual, *this);
7827
7828 if constexpr (running_in_debug_mode())
7829 {
7830 this->values_quad_initialized =
7831 evaluation_flag_actual & EvaluationFlags::values;
7832 this->gradients_quad_initialized =
7833 evaluation_flag_actual & EvaluationFlags::gradients;
7834 this->hessians_quad_initialized =
7835 evaluation_flag_actual & EvaluationFlags::hessians;
7836 }
7837 }
7838
7839
7840
7841 template <int dim,
7842 int fe_degree,
7843 int n_q_points_1d,
7844 int n_components_,
7845 typename Number,
7846 typename VectorizedArrayType>
7847 inline void
7848 FEFaceEvaluation<dim,
7849 fe_degree,
7850 n_q_points_1d,
7851 n_components_,
7852 Number,
7853 VectorizedArrayType>::
7854 integrate(const EvaluationFlags::EvaluationFlags integration_flag,
7855 const bool sum_into_values)
7856 {
7857 integrate(integration_flag, this->values_dofs, sum_into_values);
7858
7859 if constexpr (running_in_debug_mode())
7860 {
7861 this->dof_values_initialized = true;
7862 }
7863 }
7864
7865
7866
7867 template <int dim,
7868 int fe_degree,
7869 int n_q_points_1d,
7870 int n_components_,
7871 typename Number,
7872 typename VectorizedArrayType>
7873 inline void
7874 FEFaceEvaluation<dim,
7875 fe_degree,
7876 n_q_points_1d,
7877 n_components_,
7878 Number,
7879 VectorizedArrayType>::
7880 integrate(const EvaluationFlags::EvaluationFlags integration_flag,
7881 VectorizedArrayType *values_array,
7882 const bool sum_into_values)
7883 {
7884 Assert((integration_flag &
7885 ~(EvaluationFlags::values | EvaluationFlags::gradients |
7886 EvaluationFlags::hessians)) == 0,
7887 ExcMessage("Only EvaluationFlags::values, EvaluationFlags::gradients, "
7888 "and EvaluationFlags::hessians are supported."));
7889
7890 EvaluationFlags::EvaluationFlags integration_flag_actual = integration_flag;
7891 if (integration_flag & EvaluationFlags::hessians &&
7892 (this->cell_type > internal::MatrixFreeFunctions::affine))
7893 {
7894 unsigned int size = n_components * dim * n_q_points;
7895 if ((integration_flag & EvaluationFlags::gradients) != 0u)
7896 {
7897 for (unsigned int i = 0; i < size; ++i)
7898 this->gradients_quad[i] += this->gradients_from_hessians_quad[i];
7899 }
7900 else
7901 {
7902 for (unsigned int i = 0; i < size; ++i)
7903 this->gradients_quad[i] = this->gradients_from_hessians_quad[i];
7904 integration_flag_actual |= EvaluationFlags::gradients;
7905 }
7906 }
7907
7908 if (this->data->element_type ==
7909 internal::MatrixFreeFunctions::ElementType::tensor_raviart_thomas &&
7910 integration_flag & EvaluationFlags::gradients &&
7911 this->cell_type > internal::MatrixFreeFunctions::affine &&
7912 this->divergence_is_requested == false)
7913 {
7914 unsigned int size = n_components * n_q_points;
7915 if ((integration_flag & EvaluationFlags::values) != 0u)
7916 {
7917 for (unsigned int i = 0; i < size; ++i)
7918 this->values_quad[i] += this->values_from_gradients_quad[i];
7919 }
7920 else
7921 {
7922 for (unsigned int i = 0; i < size; ++i)
7923 this->values_quad[i] = this->values_from_gradients_quad[i];
7924 integration_flag_actual |= EvaluationFlags::values;
7925 }
7926 }
7927
7928 if constexpr (fe_degree > -1)
7929 internal::FEFaceEvaluationImplIntegrateSelector<dim, VectorizedArrayType>::
7930 template run<fe_degree, n_q_points_1d>(n_components,
7931 integration_flag_actual,
7932 values_array,
7933 *this,
7934 sum_into_values);
7935 else
7936 internal::FEFaceEvaluationFactory<dim, VectorizedArrayType>::integrate(
7937 n_components,
7938 integration_flag_actual,
7939 values_array,
7940 *this,
7941 sum_into_values);
7942 }
7943
7944
7945
7946 template <int dim,
7947 int fe_degree,
7948 int n_q_points_1d,
7949 int n_components_,
7950 typename Number,
7951 typename VectorizedArrayType>
7952 inline void
7953 FEFaceEvaluation<dim,
7954 fe_degree,
7955 n_q_points_1d,
7956 n_components_,
7957 Number,
7958 VectorizedArrayType>::
7959 integrate_in_face(const EvaluationFlags::EvaluationFlags integration_flag)
7960 {
7961 Assert((integration_flag &
7962 ~(EvaluationFlags::values | EvaluationFlags::gradients |
7963 EvaluationFlags::hessians)) == 0,
7964 ExcMessage("Only EvaluationFlags::values, EvaluationFlags::gradients, "
7965 "and EvaluationFlags::hessians are supported."));
7966
7967 EvaluationFlags::EvaluationFlags integration_flag_actual = integration_flag;
7968 if (integration_flag & EvaluationFlags::hessians &&
7969 (this->cell_type > internal::MatrixFreeFunctions::affine))
7970 {
7971 unsigned int size = n_components * dim * n_q_points;
7972 if ((integration_flag & EvaluationFlags::gradients) != 0u)
7973 {
7974 for (unsigned int i = 0; i < size; ++i)
7975 this->gradients_quad[i] += this->gradients_from_hessians_quad[i];
7976 }
7977 else
7978 {
7979 for (unsigned int i = 0; i < size; ++i)
7980 this->gradients_quad[i] = this->gradients_from_hessians_quad[i];
7981 integration_flag_actual |= EvaluationFlags::gradients;
7982 }
7983 }
7984
7985 if (this->data->element_type ==
7986 internal::MatrixFreeFunctions::ElementType::tensor_raviart_thomas &&
7987 integration_flag & EvaluationFlags::gradients &&
7988 this->cell_type > internal::MatrixFreeFunctions::affine &&
7989 this->divergence_is_requested == false)
7990 {
7991 unsigned int size = n_components * n_q_points;
7992 if ((integration_flag & EvaluationFlags::values) != 0u)
7993 {
7994 for (unsigned int i = 0; i < size; ++i)
7995 this->values_quad[i] += this->values_from_gradients_quad[i];
7996 }
7997 else
7998 {
7999 for (unsigned int i = 0; i < size; ++i)
8000 this->values_quad[i] = this->values_from_gradients_quad[i];
8001 integration_flag_actual |= EvaluationFlags::values;
8002 }
8003 }
8004
8005 if constexpr (fe_degree > -1)
8006 internal::FEFaceEvaluationImplIntegrateInFaceSelector<
8007 dim,
8008 VectorizedArrayType>::template run<fe_degree>(n_components,
8009 integration_flag_actual,
8010 *this);
8011 else
8012 internal::FEFaceEvaluationFactory<dim, VectorizedArrayType>::
8013 integrate_in_face(n_components, integration_flag_actual, *this);
8014
8015 // face dofs initialized
8016 }
8017
8018
8019
8020 template <int dim,
8021 int fe_degree,
8022 int n_q_points_1d,
8023 int n_components_,
8024 typename Number,
8025 typename VectorizedArrayType>
8026 inline void
8027 FEFaceEvaluation<dim,
8028 fe_degree,
8029 n_q_points_1d,
8030 n_components_,
8031 Number,
8032 VectorizedArrayType>::
8033 collect_from_face(const EvaluationFlags::EvaluationFlags integration_flag,
8034 const bool sum_into_values)
8035 {
8036 collect_from_face(integration_flag, this->values_dofs, sum_into_values);
8037
8038 if constexpr (running_in_debug_mode())
8039 {
8040 this->dof_values_initialized = true;
8041 }
8042 }
8043
8044
8045
8046 template <int dim,
8047 int fe_degree,
8048 int n_q_points_1d,
8049 int n_components_,
8050 typename Number,
8051 typename VectorizedArrayType>
8052 inline void
8053 FEFaceEvaluation<dim,
8054 fe_degree,
8055 n_q_points_1d,
8056 n_components_,
8057 Number,
8058 VectorizedArrayType>::
8059 collect_from_face(const EvaluationFlags::EvaluationFlags integration_flag,
8060 VectorizedArrayType *values_array,
8061 const bool sum_into_values)
8062 {
8063 Assert((integration_flag &
8064 ~(EvaluationFlags::values | EvaluationFlags::gradients |
8065 EvaluationFlags::hessians)) == 0,
8066 ExcMessage("Only EvaluationFlags::values, EvaluationFlags::gradients, "
8067 "and EvaluationFlags::hessians are supported."));
8068
8069 EvaluationFlags::EvaluationFlags integration_flag_actual = integration_flag;
8070 if (integration_flag & EvaluationFlags::hessians &&
8071 (this->cell_type > internal::MatrixFreeFunctions::affine))
8072 integration_flag_actual |= EvaluationFlags::gradients;
8073
8074 if (this->data->element_type ==
8075 internal::MatrixFreeFunctions::ElementType::tensor_raviart_thomas &&
8076 integration_flag & EvaluationFlags::gradients &&
8077 this->cell_type > internal::MatrixFreeFunctions::affine &&
8078 this->divergence_is_requested == false)
8079 integration_flag_actual |= EvaluationFlags::values;
8080
8081 if constexpr (fe_degree > -1)
8082 internal::FEFaceEvaluationImplCollectFromFaceSelector<
8083 dim,
8084 VectorizedArrayType>::template run<fe_degree>(n_components,
8085 integration_flag_actual,
8086 values_array,
8087 *this,
8088 sum_into_values);
8089 else
8090 internal::FEFaceEvaluationFactory<dim, VectorizedArrayType>::
8091 collect_from_face(n_components,
8092 integration_flag_actual,
8093 values_array,
8094 *this,
8095 sum_into_values);
8096 }
8097
8098
8099
8100 template <int dim,
8101 int fe_degree,
8102 int n_q_points_1d,
8103 int n_components_,
8104 typename Number,
8105 typename VectorizedArrayType>
8106 template <typename VectorType>
8107 inline void
8108 FEFaceEvaluation<dim,
8109 fe_degree,
8110 n_q_points_1d,
8111 n_components_,
8112 Number,
8113 VectorizedArrayType>::
8114 gather_evaluate(const VectorType &input_vector,
8115 const EvaluationFlags::EvaluationFlags evaluation_flag)
8116 {
8117 Assert((evaluation_flag &
8118 ~(EvaluationFlags::values | EvaluationFlags::gradients |
8119 EvaluationFlags::hessians)) == 0,
8120 ExcMessage("Only EvaluationFlags::values, EvaluationFlags::gradients, "
8121 "and EvaluationFlags::hessians are supported."));
8122
8123 const auto shared_vector_data = internal::get_shared_vector_data(
8124 &input_vector,
8125 this->dof_access_index ==
8126 internal::MatrixFreeFunctions::DoFInfo::dof_access_cell,
8127 this->active_fe_index,
8128 this->dof_info);
8129
8130 if (this->data->data.front().fe_degree > 0 &&
8131 fast_evaluation_supported(this->data->data.front().fe_degree,
8132 this->data->data.front().n_q_points_1d) &&
8133 internal::FEFaceEvaluationImplGatherEvaluateSelector<
8134 dim,
8135 typename VectorType::value_type,
8136 VectorizedArrayType>::
8137 supports(evaluation_flag,
8138 *this->data,
8139 internal::get_beginning<typename VectorType::value_type>(
8140 input_vector),
8141 this->dof_info->index_storage_variants[this->dof_access_index]
8142 [this->cell]))
8143 {
8144 if constexpr (fe_degree > -1)
8145 {
8146 internal::FEFaceEvaluationImplGatherEvaluateSelector<
8147 dim,
8148 typename VectorType::value_type,
8149 VectorizedArrayType>::template run<fe_degree,
8150 n_q_points_1d>(
8151 n_components,
8152 evaluation_flag,
8153 internal::get_beginning<typename VectorType::value_type>(
8154 input_vector),
8155 shared_vector_data,
8156 *this);
8157 }
8158 else
8159 {
8160 internal::FEFaceEvaluationGatherFactory<
8161 dim,
8162 typename VectorType::value_type,
8163 VectorizedArrayType>::evaluate(n_components,
8164 evaluation_flag,
8165 internal::get_beginning<
8166 typename VectorType::value_type>(
8167 input_vector),
8168 shared_vector_data,
8169 *this);
8170 }
8171 }
8172 else
8173 {
8174 this->read_dof_values(input_vector);
8175 this->evaluate(evaluation_flag);
8176 }
8177
8178 if constexpr (running_in_debug_mode())
8179 {
8180 this->values_quad_initialized = evaluation_flag & EvaluationFlags::values;
8181 this->gradients_quad_initialized =
8182 evaluation_flag & EvaluationFlags::gradients;
8183 this->hessians_quad_initialized =
8184 evaluation_flag & EvaluationFlags::hessians;
8185 }
8186 }
8187
8188
8189
8190 template <int dim,
8191 int fe_degree,
8192 int n_q_points_1d,
8193 int n_components_,
8194 typename Number,
8195 typename VectorizedArrayType>
8196 template <typename VectorType>
8197 inline void
8198 FEFaceEvaluation<
8199 dim,
8200 fe_degree,
8201 n_q_points_1d,
8202 n_components_,
8203 Number,
8204 VectorizedArrayType>::integrate_scatter(const bool integrate_values,
8205 const bool integrate_gradients,
8206 VectorType &destination)
8207 {
8208 const EvaluationFlags::EvaluationFlags flag =
8209 ((integrate_values) ? EvaluationFlags::values : EvaluationFlags::nothing) |
8210 ((integrate_gradients) ? EvaluationFlags::gradients :
8211 EvaluationFlags::nothing);
8212
8213 integrate_scatter(flag, destination);
8214 }
8215
8216
8217
8218 template <int dim,
8219 int fe_degree,
8220 int n_q_points_1d,
8221 int n_components_,
8222 typename Number,
8223 typename VectorizedArrayType>
8224 template <typename VectorType>
8225 inline void
8226 FEFaceEvaluation<dim,
8227 fe_degree,
8228 n_q_points_1d,
8229 n_components_,
8230 Number,
8231 VectorizedArrayType>::
8232 integrate_scatter(const EvaluationFlags::EvaluationFlags integration_flag,
8233 VectorType &destination)
8234 {
8235 Assert((this->dof_access_index ==
8236 internal::MatrixFreeFunctions::DoFInfo::dof_access_cell &&
8237 this->is_interior_face() == false) == false,
8238 ExcNotImplemented());
8239
8240 const auto shared_vector_data = internal::get_shared_vector_data(
8241 &destination,
8242 this->dof_access_index ==
8243 internal::MatrixFreeFunctions::DoFInfo::dof_access_cell,
8244 this->active_fe_index,
8245 this->dof_info);
8246
8247 if (this->data->data.front().fe_degree > 0 &&
8248 fast_evaluation_supported(this->data->data.front().fe_degree,
8249 this->data->data.front().n_q_points_1d) &&
8250 internal::FEFaceEvaluationImplGatherEvaluateSelector<
8251 dim,
8252 typename VectorType::value_type,
8253 VectorizedArrayType>::
8254 supports(integration_flag,
8255 *this->data,
8256 internal::get_beginning<typename VectorType::value_type>(
8257 destination),
8258 this->dof_info->index_storage_variants[this->dof_access_index]
8259 [this->cell]))
8260 {
8261 if constexpr (fe_degree > -1)
8262 {
8263 internal::FEFaceEvaluationImplIntegrateScatterSelector<
8264 dim,
8265 typename VectorType::value_type,
8266 VectorizedArrayType>::template run<fe_degree,
8267 n_q_points_1d>(
8268 n_components,
8269 integration_flag,
8270 internal::get_beginning<typename VectorType::value_type>(
8271 destination),
8272 shared_vector_data,
8273 *this);
8274 }
8275 else
8276 {
8277 internal::FEFaceEvaluationGatherFactory<
8278 dim,
8279 typename VectorType::value_type,
8280 VectorizedArrayType>::integrate(n_components,
8281 integration_flag,
8282 internal::get_beginning<
8283 typename VectorType::value_type>(
8284 destination),
8285 shared_vector_data,
8286 *this);
8287 }
8288 }
8289 else
8290 {
8291 integrate(integration_flag);
8292 this->distribute_local_to_global(destination);
8293 }
8294 }
8295
8296
8297
8298 template <int dim,
8299 int fe_degree,
8300 int n_q_points_1d,
8301 int n_components_,
8302 typename Number,
8303 typename VectorizedArrayType>
8304 inline std_cxx20::ranges::iota_view<unsigned int, unsigned int>
8305 FEFaceEvaluation<dim,
8306 fe_degree,
8307 n_q_points_1d,
8308 n_components_,
8309 Number,
8310 VectorizedArrayType>::dof_indices() const
8311 {
8312 return std_cxx20::ranges::iota_view<unsigned int, unsigned int>(
8313 0U, dofs_per_cell);
8314 }
8315
8316
8317
8318 template <int dim,
8319 int fe_degree,
8320 int n_q_points_1d,
8321 int n_components_,
8322 typename Number,
8323 typename VectorizedArrayType>
8324 bool
8325 FEEvaluation<dim,
8326 fe_degree,
8327 n_q_points_1d,
8328 n_components_,
8329 Number,
8330 VectorizedArrayType>::
8331 fast_evaluation_supported(const unsigned int given_degree,
8332 const unsigned int given_n_q_points_1d)
8333 {
8334 return fe_degree == -1 ?
8335 internal::FEEvaluationFactory<dim, VectorizedArrayType>::
8336 fast_evaluation_supported(given_degree, given_n_q_points_1d) :
8337 true;
8338 }
8339
8340
8341
8342 template <int dim,
8343 int fe_degree,
8344 int n_q_points_1d,
8345 int n_components_,
8346 typename Number,
8347 typename VectorizedArrayType>
8348 bool
8349 FEFaceEvaluation<dim,
8350 fe_degree,
8351 n_q_points_1d,
8352 n_components_,
8353 Number,
8354 VectorizedArrayType>::
8355 fast_evaluation_supported(const unsigned int given_degree,
8356 const unsigned int given_n_q_points_1d)
8357 {
8358 return fe_degree == -1 ?
8359 internal::FEFaceEvaluationFactory<dim, VectorizedArrayType>::
8360 fast_evaluation_supported(given_degree, given_n_q_points_1d) :
8361 true;
8362 }
8363
8364
8365
8366 template <int dim,
8367 int fe_degree,
8368 int n_q_points_1d,
8369 int n_components_,
8370 typename Number,
8371 typename VectorizedArrayType>
8372 bool
8373 FEFaceEvaluation<dim,
8374 fe_degree,
8375 n_q_points_1d,
8376 n_components_,
8377 Number,
8378 VectorizedArrayType>::at_boundary() const
8379 {
8380 Assert(this->dof_access_index !=
8381 internal::MatrixFreeFunctions::DoFInfo::dof_access_cell,
8382 ExcNotImplemented());
8383
8384 if (this->is_interior_face() == false)
8385 return false;
8386 else if (this->cell < this->matrix_free->n_inner_face_batches())
8387 return false;
8388 else if (this->cell < (this->matrix_free->n_inner_face_batches() +
8389 this->matrix_free->n_boundary_face_batches()))
8390 return true;
8391 else
8392 return false;
8393 }
8394
8395
8396
8397 template <int dim,
8398 int fe_degree,
8399 int n_q_points_1d,
8400 int n_components_,
8401 typename Number,
8402 typename VectorizedArrayType>
8403 types::boundary_id
8404 FEFaceEvaluation<dim,
8405 fe_degree,
8406 n_q_points_1d,
8407 n_components_,
8408 Number,
8409 VectorizedArrayType>::boundary_id() const
8410 {
8411 Assert(this->dof_access_index !=
8412 internal::MatrixFreeFunctions::DoFInfo::dof_access_cell,
8413 ExcNotImplemented());
8414
8415 if (at_boundary())
8416 return this->matrix_free->get_boundary_id(this->cell);
8417 else
8418 return numbers::internal_face_boundary_id;
8419 }
8420
8421
8422
8423 template <int dim,
8424 int fe_degree,
8425 int n_q_points_1d,
8426 int n_components_,
8427 typename Number,
8428 typename VectorizedArrayType>
8429 unsigned int
8430 FEFaceEvaluation<
8431 dim,
8432 fe_degree,
8433 n_q_points_1d,
8434 n_components_,
8435 Number,
8436 VectorizedArrayType>::get_dofs_per_component_projected_to_face()
8437 {
8438 return this->data->dofs_per_component_on_face;
8439 }
8440
8441
8442
8443 template <int dim,
8444 int fe_degree,
8445 int n_q_points_1d,
8446 int n_components_,
8447 typename Number,
8448 typename VectorizedArrayType>
8449 unsigned int
8450 FEFaceEvaluation<dim,
8451 fe_degree,
8452 n_q_points_1d,
8453 n_components_,
8454 Number,
8455 VectorizedArrayType>::get_dofs_projected_to_face()
8456 {
8457 return this->data->dofs_per_component_on_face * n_components_;
8458 }
8459
8460
8461
8462 /*------------------------- end FEFaceEvaluation ------------------------- */
8463
8464
8465 #endif // ifndef DOXYGEN
8466
8467
8468 DEAL_II_NAMESPACE_CLOSE
8469
8470 #endif