Debian Codegraph @

include/deal.II/matrix_free/mapping_info.templates.h

raw
1 // ------------------------------------------------------------------------
2 //
3 // SPDX-License-Identifier: LGPL-2.1-or-later
4 // Copyright (C) 2012 - 2025 by the deal.II authors
5 //
6 // This file is part of the deal.II library.
7 //
8 // Part of the source code is dual licensed under Apache-2.0 WITH
9 // LLVM-exception OR LGPL-2.1-or-later. Detailed license information
10 // governing the source code and code contributions can be found in
11 // LICENSE.md and CONTRIBUTING.md at the top level directory of deal.II.
12 //
13 // ------------------------------------------------------------------------
14
15 #ifndef dealii_matrix_free_mapping_info_templates_h
16 #define dealii_matrix_free_mapping_info_templates_h
17
18 #include <deal.II/base/config.h>
19
20 #include <deal.II/base/floating_point_comparator.h>
21 #include <deal.II/base/memory_consumption.h>
22 #include <deal.II/base/multithread_info.h>
23 #include <deal.II/base/thread_management.h>
24 #include <deal.II/base/utilities.h>
25
26 #include <deal.II/fe/fe_dgq.h>
27 #include <deal.II/fe/fe_nothing.h>
28 #include <deal.II/fe/fe_values.h>
29 #include <deal.II/fe/mapping_q.h>
30
31 #include <deal.II/matrix_free/evaluation_template_factory.h>
32 #include <deal.II/matrix_free/fe_evaluation_data.h>
33 #include <deal.II/matrix_free/mapping_info.h>
34 #include <deal.II/matrix_free/mapping_info_storage.templates.h>
35 #include <deal.II/matrix_free/util.h>
36
37 #include <limits>
38
39 DEAL_II_NAMESPACE_OPEN
40
41
42 namespace internal
43 {
44 namespace MatrixFreeFunctions
45 {
46 template <int dim, typename Number, typename VectorizedArrayType>
47 void
48 MappingInfo<dim, Number, VectorizedArrayType>::clear()
49 {
50 cell_data.clear();
51 face_data.clear();
52 face_data_by_cells.clear();
53 cell_type.clear();
54 face_type.clear();
55 mapping_collection = nullptr;
56 mapping = nullptr;
57 }
58
59
60
61 template <int dim, typename Number, typename VectorizedArrayType>
62 void
63 MappingInfo<dim, Number, VectorizedArrayType>::initialize(
64 const dealii::Triangulation<dim> &tria,
65 const std::vector<std::pair<unsigned int, unsigned int>> &cells,
66 const FaceInfo<VectorizedArrayType::size()> &face_info,
67 const std::vector<unsigned int> &active_fe_index,
68 const std::shared_ptr<dealii::hp::MappingCollection<dim>> &mapping,
69 const std::vector<dealii::hp::QCollection<dim>> &quad,
70 const UpdateFlags update_flags_cells,
71 const UpdateFlags update_flags_boundary_faces,
72 const UpdateFlags update_flags_inner_faces,
73 const UpdateFlags update_flags_faces_by_cells,
74 const bool piola_transform)
75 {
76 clear();
77 this->mapping_collection = mapping;
78 this->mapping = &mapping->operator[](0);
79
80 cell_data.resize(quad.size());
81 face_data.resize(quad.size());
82 face_data_by_cells.resize(quad.size());
83
84 const bool is_mixed_mesh = tria.is_mixed_mesh();
85
86 // dummy FE that is used to set up an FEValues object. Do not need the
87 // actual finite element because we will only evaluate quantities for
88 // the mapping that are independent of the FE
89 this->update_flags_cells =
90 MappingInfoStorage<dim, dim, VectorizedArrayType>::compute_update_flags(
91 update_flags_cells, quad, piola_transform);
92
93 this->update_flags_boundary_faces =
94 ((update_flags_inner_faces | update_flags_boundary_faces) &
95 update_quadrature_points ?
96 update_quadrature_points :
97 update_default) |
98 ((((update_flags_inner_faces | update_flags_boundary_faces) &
99 (update_jacobian_grads | update_hessians)) != 0u ||
100 (piola_transform &&
101 ((update_flags_inner_faces | update_flags_boundary_faces) &
102 (update_gradients | update_contravariant_transformation)) != 0u)) ?
103 update_jacobian_grads :
104 update_default) |
105 update_normal_vectors | update_JxW_values | update_jacobians;
106 this->update_flags_inner_faces = this->update_flags_boundary_faces;
107 this->update_flags_faces_by_cells = update_flags_faces_by_cells;
108
109 reference_cell_types.resize(quad.size());
110
111 for (unsigned int my_q = 0; my_q < quad.size(); ++my_q)
112 {
113 const unsigned int n_hp_quads = quad[my_q].size();
114 AssertIndexRange(0, n_hp_quads);
115
116 const unsigned int scale = std::max<unsigned int>(1, dim - 1);
117
118 cell_data[my_q].descriptor.resize(n_hp_quads);
119 face_data[my_q].descriptor.resize(n_hp_quads * scale);
120 face_data[my_q].q_collection.resize(n_hp_quads);
121 face_data_by_cells[my_q].descriptor.resize(n_hp_quads * scale);
122 reference_cell_types[my_q].resize(n_hp_quads);
123
124 for (unsigned int hpq = 0; hpq < n_hp_quads; ++hpq)
125 {
126 bool flag = quad[my_q][hpq].is_tensor_product();
127
128 if (flag)
129 for (unsigned int i = 1; i < dim; ++i)
130 flag &= quad[my_q][hpq].get_tensor_basis()[0] ==
131 quad[my_q][hpq].get_tensor_basis()[i];
132
133 if (flag == false)
134 {
135 cell_data[my_q].descriptor[hpq].initialize(quad[my_q][hpq]);
136 const auto quad_face =
137 get_unique_face_quadratures(quad[my_q][hpq]);
138
139 if (quad_face.first.size() > 0) // line, quad
140 {
141 face_data[my_q].descriptor[hpq * scale].initialize(
142 quad_face.first);
143 face_data_by_cells[my_q]
144 .descriptor[hpq * scale]
145 .initialize(quad_face.first);
146 }
147
148 if (quad_face.second.size() > 0) // triangle
149 {
150 AssertDimension(dim, 3);
151 face_data[my_q]
152 .descriptor[hpq * scale + (is_mixed_mesh ? 1 : 0)]
153 .initialize(quad_face.second);
154 face_data_by_cells[my_q]
155 .descriptor[hpq * scale + (is_mixed_mesh ? 1 : 0)]
156 .initialize(quad_face.second);
157 }
158
159 const auto face_quadrature_collection =
160 get_face_quadrature_collection(quad[my_q][hpq]);
161
162 face_data[my_q].q_collection[hpq] =
163 face_quadrature_collection.second;
164
165 reference_cell_types[my_q][hpq] =
166 face_quadrature_collection.first;
167 }
168 else
169 {
170 cell_data[my_q].descriptor[hpq].initialize(
171 quad[my_q][hpq].get_tensor_basis()[0]);
172
173 const auto quad_face = quad[my_q][hpq].get_tensor_basis()[0];
174 face_data[my_q].descriptor[hpq * scale].initialize(quad_face);
175 face_data[my_q].q_collection[hpq] =
176 dealii::hp::QCollection<dim - 1>(quad_face);
177 face_data_by_cells[my_q].descriptor[hpq * scale].initialize(
178 quad_face);
179 reference_cell_types[my_q][hpq] =
180 ReferenceCells::get_hypercube<dim>();
181 }
182 }
183 }
184
185 // In case we have no hp-adaptivity (active_fe_index is empty), we have
186 // cells, and the mapping is MappingQ or a derived class, we can
187 // use the fast method.
188 if (active_fe_index.empty() && !cells.empty() && mapping->size() == 1 &&
189 dynamic_cast<const MappingQ<dim> *>(&mapping->operator[](0)))
190 compute_mapping_q(tria, cells, face_info);
191 else
192 {
193 // Could call these functions in parallel, but not useful because
194 // the work inside is nicely split up already
195 initialize_cells(tria, cells, active_fe_index, *mapping);
196 initialize_faces(
197 tria, cells, face_info.faces, active_fe_index, *mapping);
198 initialize_faces_by_cells(tria, cells, face_info, *mapping);
199 }
200 }
201
202
203
204 template <int dim, typename Number, typename VectorizedArrayType>
205 void
206 MappingInfo<dim, Number, VectorizedArrayType>::update_mapping(
207 const dealii::Triangulation<dim> &tria,
208 const std::vector<std::pair<unsigned int, unsigned int>> &cells,
209 const FaceInfo<VectorizedArrayType::size()> &face_info,
210 const std::vector<unsigned int> &active_fe_index,
211 const std::shared_ptr<dealii::hp::MappingCollection<dim>> &mapping)
212 {
213 AssertDimension(cells.size() / VectorizedArrayType::size(),
214 cell_type.size());
215
216 for (auto &data : cell_data)
217 data.clear_data_fields();
218 for (auto &data : face_data)
219 data.clear_data_fields();
220 for (auto &data : face_data_by_cells)
221 data.clear_data_fields();
222
223 this->mapping_collection = mapping;
224 this->mapping = &mapping->operator[](0);
225
226 if (active_fe_index.empty() && !cells.empty() && mapping->size() == 1 &&
227 dynamic_cast<const MappingQ<dim> *>(&mapping->operator[](0)))
228 compute_mapping_q(tria, cells, face_info);
229 else
230 {
231 // Could call these functions in parallel, but not useful because
232 // the work inside is nicely split up already
233 initialize_cells(tria, cells, active_fe_index, *mapping);
234 initialize_faces(
235 tria, cells, face_info.faces, active_fe_index, *mapping);
236 initialize_faces_by_cells(tria, cells, face_info, *mapping);
237 }
238 }
239
240
241
242 // Copy a vectorized array of one type to another type
243 template <typename VectorizedArrayType1, typename VectorizedArrayType2>
244 inline DEAL_II_ALWAYS_INLINE void
245 store_vectorized_array(const VectorizedArrayType1 value,
246 const unsigned int offset,
247 VectorizedArrayType2 &result)
248 {
249 static_assert(VectorizedArrayType2::size() >=
250 VectorizedArrayType1::size(),
251 "Cannot convert to vectorized array of wider number type");
252
253 DEAL_II_OPENMP_SIMD_PRAGMA
254 for (unsigned int v = 0; v < VectorizedArrayType1::size(); ++v)
255 result[offset + v] = value[v];
256 }
257
258
259
260 // For second derivatives on the real cell, we need the gradient of the
261 // inverse Jacobian J. This involves some calculus and is done
262 // vectorized. If L is the gradient of the Jacobian on the unit cell,
263 // the gradient of the inverse is given by (multidimensional calculus) -
264 // J * (J * L) * J (the third J is because we need to transform the
265 // gradient L from the unit to the real cell, and then apply the inverse
266 // Jacobian). Compare this with 1d with j(x) = 1/k(phi(x)), where j =
267 // phi' is the inverse of the Jacobian and k is the derivative of the
268 // Jacobian on the unit cell. Then j' = phi' k'/k^2 = j k' j^2.
269 template <int dim, typename Number>
270 Tensor<1, dim *(dim + 1) / 2, Tensor<1, dim, Number>>
271 process_jacobian_gradient(const Tensor<2, dim, Number> &inv_jac_permutation,
272 const Tensor<2, dim, Number> &inv_jac,
273 const Tensor<3, dim, Number> &jac_grad)
274 {
275 Number inv_jac_grad[dim][dim][dim];
276
277 // compute: inv_jac_grad = inv_jac_permutation * grad_unit(jac)
278 for (unsigned int d = 0; d < dim; ++d)
279 for (unsigned int e = 0; e < dim; ++e)
280 for (unsigned int f = 0; f < dim; ++f)
281 {
282 inv_jac_grad[f][e][d] =
283 (inv_jac_permutation[f][0] * jac_grad[d][e][0]);
284 for (unsigned int g = 1; g < dim; ++g)
285 inv_jac_grad[f][e][d] +=
286 (inv_jac_permutation[f][g] * jac_grad[d][e][g]);
287 }
288
289 // compute: transpose (-inv_jac_permutation * inv_jac_grad[d] * inv_jac)
290 Number tmp[dim];
291 Number grad_jac_inv[dim][dim][dim];
292 for (unsigned int d = 0; d < dim; ++d)
293 for (unsigned int e = 0; e < dim; ++e)
294 {
295 for (unsigned int f = 0; f < dim; ++f)
296 {
297 tmp[f] = -inv_jac_grad[d][f][0] * inv_jac[0][e];
298 for (unsigned int g = 1; g < dim; ++g)
299 tmp[f] -= inv_jac_grad[d][f][g] * inv_jac[g][e];
300 }
301
302 // needed for non-diagonal part of Jacobian grad
303 for (unsigned int f = 0; f < dim; ++f)
304 {
305 grad_jac_inv[f][d][e] = inv_jac_permutation[f][0] * tmp[0];
306 for (unsigned int g = 1; g < dim; ++g)
307 grad_jac_inv[f][d][e] += inv_jac_permutation[f][g] * tmp[g];
308 }
309 }
310
311 Tensor<1, dim *(dim + 1) / 2, Tensor<1, dim, Number>> result;
312
313 // the diagonal part of Jacobian gradient comes first
314 for (unsigned int d = 0; d < dim; ++d)
315 for (unsigned int f = 0; f < dim; ++f)
316 result[d][f] = grad_jac_inv[d][d][f];
317
318 // then the upper-diagonal part
319 for (unsigned int d = 0, count = 0; d < dim; ++d)
320 for (unsigned int e = d + 1; e < dim; ++e, ++count)
321 for (unsigned int f = 0; f < dim; ++f)
322 result[dim + count][f] = grad_jac_inv[d][e][f];
323 return result;
324 }
325
326
327
328 /* ------------------------- initialization of cells ------------------- */
329
330 // Namespace with implementation of extraction of values on cell
331 // range
332 namespace ExtractCellHelper
333 {
334 template <int dim>
335 double
336 get_jacobian_size(const dealii::Triangulation<dim> &tria)
337 {
338 if (tria.n_cells() == 0)
339 return 1;
340 else
341 return tria.begin()->diameter();
342 }
343
344
345
346 template <int dim, typename Number, typename VectorizedArrayType>
347 struct CompressedCellData
348 {
349 CompressedCellData(const double expected_size)
350 : data(FloatingPointComparator<VectorizedArrayType>(
351 expected_size * std::numeric_limits<double>::epsilon() * 1024.))
352 {}
353
354 std::map<Tensor<2, dim, Tensor<1, VectorizedArrayType::size(), Number>>,
355 unsigned int,
356 FloatingPointComparator<VectorizedArrayType>>
357 data;
358 };
359
360 /**
361 * Internal temporary data used for the initialization.
362 */
363 template <int dim, typename Number, typename VectorizedArrayType>
364 struct LocalData
365 {
366 LocalData(const double jac_size);
367 void
368 resize(const unsigned int size);
369
370 AlignedVector<Point<dim, VectorizedArrayType>> quadrature_points;
371 AlignedVector<Tensor<2, dim, VectorizedArrayType>> general_jac;
372 AlignedVector<VectorizedArrayType> JxW_values;
373 AlignedVector<Tensor<3, dim, VectorizedArrayType>> general_jac_grad;
374 AlignedVector<Tensor<1, dim, VectorizedArrayType>> normal_vectors;
375 Tensor<2, dim, VectorizedArrayType> const_jac;
376 const double jac_size;
377 };
378
379
380
381 template <int dim, typename Number, typename VectorizedArrayType>
382 LocalData<dim, Number, VectorizedArrayType>::LocalData(
383 const double jac_size_in)
384 : jac_size(jac_size_in)
385 {}
386
387
388
389 template <int dim, typename Number, typename VectorizedArrayType>
390 void
391 LocalData<dim, Number, VectorizedArrayType>::resize(
392 const unsigned int size)
393 {
394 if (JxW_values.size() != size)
395 {
396 quadrature_points.resize_fast(size);
397 general_jac.resize_fast(size * 2);
398 JxW_values.resize_fast(size);
399 general_jac_grad.resize_fast(size * 2);
400 normal_vectors.resize_fast(size);
401 }
402 }
403
404
405
406 /**
407 * Helper function called internally during the initialize function.
408 */
409 template <int dim, typename VectorizedArrayType>
410 void
411 evaluate_on_cell(const dealii::Triangulation<dim> &tria,
412 const std::pair<unsigned int, unsigned int> *cells,
413 const unsigned int my_q,
414 GeometryType &cell_t_prev,
415 GeometryType *cell_t,
416 dealii::FEValues<dim, dim> &fe_val,
417 LocalData<dim,
418 typename VectorizedArrayType::value_type,
419 VectorizedArrayType> &cell_data)
420 {
421 const unsigned int n_q_points = fe_val.n_quadrature_points;
422 const UpdateFlags update_flags = fe_val.get_update_flags();
423
424 cell_data.const_jac = Tensor<2, dim, VectorizedArrayType>();
425
426 // this should be the same value as used in HashValue::scaling (but we
427 // not have that field here)
428 const double zero_tolerance_double =
429 cell_data.jac_size * std::numeric_limits<double>::epsilon() * 1024.;
430 for (unsigned int j = 0; j < VectorizedArrayType::size(); ++j)
431 {
432 typename dealii::Triangulation<dim>::cell_iterator cell_it(
433 &tria, cells[j].first, cells[j].second);
434 fe_val.reinit(cell_it);
435 cell_t[j] = general;
436
437 // extract quadrature points and store them temporarily. if we have
438 // Cartesian cells, we can compress the indices
439 if (update_flags & update_quadrature_points)
440 for (unsigned int q = 0; q < n_q_points; ++q)
441 {
442 const Point<dim> &point = fe_val.quadrature_point(q);
443 for (unsigned int d = 0; d < dim; ++d)
444 cell_data.quadrature_points[q][d][j] = point[d];
445 }
446
447 // if this is not the first quadrature formula and we already have
448 // determined that this cell is either Cartesian or with constant
449 // Jacobian, we have nothing more to do.
450 if (my_q > 0 && cell_t_prev <= affine)
451 continue;
452
453 // first round: if the transformation is detected to be the same as
454 // on the old cell, we only need to copy over the data.
455 if (fe_val.get_cell_similarity() == CellSimilarity::translation &&
456 my_q == 0)
457 {
458 if (j == 0)
459 cell_t[j] = cell_t_prev;
460 else
461 cell_t[j] = cell_t[j - 1];
462 }
463
464 const DerivativeForm<1, dim, dim> &jac_0 = fe_val.jacobian(0);
465
466 if (my_q == 0)
467 {
468 // check whether the Jacobian is constant on this cell the first
469 // time we come around here
470 if (cell_t[j] == general)
471 {
472 bool jacobian_constant = true;
473 for (unsigned int q = 1; q < n_q_points; ++q)
474 {
475 const DerivativeForm<1, dim, dim> &jac =
476 fe_val.jacobian(q);
477 for (unsigned int d = 0; d < dim; ++d)
478 for (unsigned int e = 0; e < dim; ++e)
479 if (std::fabs(jac_0[d][e] - jac[d][e]) >
480 zero_tolerance_double)
481 jacobian_constant = false;
482 if (jacobian_constant == false)
483 break;
484 }
485
486 // check whether the Jacobian is diagonal to machine
487 // accuracy
488 bool cell_cartesian = jacobian_constant;
489 for (unsigned int d = 0; d < dim; ++d)
490 for (unsigned int e = 0; e < dim; ++e)
491 if (d != e)
492 if (std::fabs(jac_0[d][e]) > zero_tolerance_double)
493 {
494 cell_cartesian = false;
495 break;
496 }
497
498 // in case we have only one quadrature point, we can have
499 // non-constant Jacobians, but we cannot detect it by
500 // comparison from one quadrature point to the next: in that
501 // case, need to look at second derivatives and see whether
502 // there are some non-zero entries (this is necessary since
503 // we determine the constness of the Jacobian for the first
504 // quadrature formula and might not look at them any more
505 // for the second, third quadrature formula). in any case,
506 // the flag update_jacobian_grads will be set in that case
507 if (cell_cartesian == false && n_q_points == 1 &&
508 update_flags & update_jacobian_grads)
509 {
510 const DerivativeForm<1, dim, dim> &jac =
511 fe_val.jacobian(0);
512 const DerivativeForm<2, dim, dim> &jacobian_grad =
513 fe_val.jacobian_grad(0);
514 for (unsigned int d = 0; d < dim; ++d)
515 for (unsigned int e = 0; e < dim; ++e)
516 for (unsigned int f = 0; f < dim; ++f)
517 {
518 double jac_grad_comp =
519 (jac[f][0] * jacobian_grad[d][e][0]);
520 for (unsigned int g = 1; g < dim; ++g)
521 jac_grad_comp +=
522 (jac[f][g] * jacobian_grad[d][e][g]);
523 if (std::fabs(jac_grad_comp) >
524 zero_tolerance_double)
525 jacobian_constant = false;
526 }
527 }
528 // set cell type
529 if (cell_cartesian == true)
530 cell_t[j] = cartesian;
531 else if (jacobian_constant == true)
532 cell_t[j] = affine;
533 else
534 cell_t[j] = general;
535 }
536
537 // Cartesian cell
538 if (cell_t[j] == cartesian)
539 {
540 // set Jacobian into diagonal (off-diagonal part is already
541 // zeroed out)
542 for (unsigned int d = 0; d < dim; ++d)
543 cell_data.const_jac[d][d][j] = jac_0[d][d];
544 continue;
545 }
546
547 // cell with affine mapping
548 else if (cell_t[j] == affine)
549 {
550 // compress out very small values
551 for (unsigned int d = 0; d < dim; ++d)
552 for (unsigned int e = 0; e < dim; ++e)
553 if (std::fabs(jac_0[d][e]) != 0.)
554 cell_data.const_jac[d][e][j] = jac_0[d][e];
555 continue;
556 }
557 }
558
559 // general cell case
560
561 // go through all quadrature points and fill in the data into the
562 // temporary data structures with slots for the vectorized data
563 // types
564 for (unsigned int q = 0; q < n_q_points; ++q)
565 {
566 // compress out very small numbers which are only noise. Then it
567 // is cleaner to use zero straight away (though it does not save
568 // any memory)
569 const DerivativeForm<1, dim, dim> &jac = fe_val.jacobian(q);
570 for (unsigned int d = 0; d < dim; ++d)
571 for (unsigned int e = 0; e < dim; ++e)
572 cell_data.general_jac[q][d][e][j] =
573 std::fabs(jac[d][e]) < zero_tolerance_double ? 0. :
574 jac[d][e];
575
576 // need to do some calculus based on the gradient of the
577 // Jacobian, in order to find the gradient of the inverse
578 // Jacobian which is needed in user code. however, we would like
579 // to perform that on vectorized data types instead of doubles
580 // or floats. to this end, copy the gradients first
581 if (update_flags & update_jacobian_grads)
582 {
583 const DerivativeForm<2, dim, dim> &jacobian_grad =
584 fe_val.jacobian_grad(q);
585 for (unsigned int d = 0; d < dim; ++d)
586 for (unsigned int e = 0; e < dim; ++e)
587 for (unsigned int f = 0; f < dim; ++f)
588 cell_data.general_jac_grad[q][d][e][f][j] =
589 jacobian_grad[d][e][f];
590 }
591 }
592 } // end loop over entries of vectorization (size() cells)
593
594 // set information for next cell
595 cell_t_prev = cell_t[VectorizedArrayType::size() - 1];
596 }
597
598
599
600 /**
601 * This function computes and tabulates the mapping information for
602 * generic mappings on a range of cells by a call to FEValues with the
603 * respective UpdateFlags set. There is a specialized function
604 * mapping_q_compute_range function that provides a faster
605 * initialization for mappings derived from MappingQ and suitable
606 * quadrature formulas.
607 */
608 template <int dim, typename Number, typename VectorizedArrayType>
609 void
610 initialize_cell_range(
611 const std::pair<unsigned int, unsigned int> cell_range,
612 const dealii::Triangulation<dim> &tria,
613 const std::vector<std::pair<unsigned int, unsigned int>> &cells,
614 const std::vector<unsigned int> &active_fe_index,
615 const dealii::hp::MappingCollection<dim> &mapping,
616 MappingInfo<dim, Number, VectorizedArrayType> &mapping_info,
617 std::pair<
618 std::vector<MappingInfoStorage<dim, dim, VectorizedArrayType>>,
619 CompressedCellData<dim, Number, VectorizedArrayType>> &data)
620 {
621 FE_Nothing<dim> dummy_fe;
622
623 // when we make comparisons about the size of Jacobians we need to
624 // know the approximate size of typical entries in Jacobians. We need
625 // to fix the Jacobian size once and for all. We choose the diameter
626 // of the first cell (on level zero, which is the best accuracy we can
627 // hope for, since diameters on finer levels are computed by
628 // differences of nearby cells) as the order of magnitude by which we
629 // make comparisons "relative."
630 const double jacobian_size = get_jacobian_size(tria);
631
632 // objects that hold the data for up to vectorization_width cells while
633 // we fill them up. Only after all vectorization_width cells have been
634 // processed, we can insert the data into the data structures of this
635 // class
636 LocalData<dim, Number, VectorizedArrayType> cell_data(jacobian_size);
637
638 // encodes the cell types of the current cell. Since several cells
639 // must be considered together, this variable holds the individual
640 // info of the last chunk of cells
641 GeometryType cell_t[VectorizedArrayType::size()];
642 GeometryType cell_t_prev = general;
643
644 // fe_values object that is used to compute the mapping data. for
645 // the hp-case there might be more than one finite element. since we
646 // manually select the active FE index and not via a
647 // DoFHandler<dim>::active_cell_iterator, we need to manually
648 // select the correct finite element, so just hold a vector of
649 // FEValues
650 std::vector<std::vector<std::shared_ptr<dealii::FEValues<dim>>>>
651 fe_values(mapping_info.cell_data.size());
652
653 const unsigned int max_active_fe_index =
654 active_fe_index.size() > 0 ?
655 *std::max_element(active_fe_index.begin(), active_fe_index.end()) :
656 0;
657
658 for (unsigned int i = 0; i < fe_values.size(); ++i)
659 fe_values[i].resize(max_active_fe_index + 1);
660 const UpdateFlags update_flags = mapping_info.update_flags_cells;
661 const UpdateFlags update_flags_feval =
662 (update_flags & update_jacobians ? update_jacobians :
663 update_default) |
664 (update_flags & update_jacobian_grads ? update_jacobian_grads :
665 update_default) |
666 (update_flags & update_quadrature_points ? update_quadrature_points :
667 update_default);
668
669 const unsigned int end_cell = std::min(mapping_info.cell_type.size(),
670 std::size_t(cell_range.second));
671 // loop over given cells
672 for (unsigned int cell = cell_range.first; cell < end_cell; ++cell)
673 for (unsigned int my_q = 0; my_q < mapping_info.cell_data.size();
674 ++my_q)
675 {
676 // GENERAL OUTLINE: First generate the data in format "number"
677 // for vectorization_width cells, and then find the most
678 // general type of cell for appropriate vectorized formats. then
679 // fill this data in
680 const unsigned int fe_index =
681 active_fe_index.size() > 0 ? active_fe_index[cell] : 0;
682 const unsigned int hp_quad_index =
683 mapping_info.cell_data[my_q].descriptor.size() == 1 ? 0 :
684 fe_index;
685 const unsigned int hp_mapping_index =
686 mapping.size() == 1 ? 0 : fe_index;
687 const unsigned int n_q_points = mapping_info.cell_data[my_q]
688 .descriptor[hp_quad_index]
689 .n_q_points;
690 if (fe_values[my_q][fe_index].get() == nullptr)
691 fe_values[my_q][fe_index] =
692 std::make_shared<dealii::FEValues<dim>>(
693 mapping[hp_mapping_index],
694 dummy_fe,
695 mapping_info.cell_data[my_q]
696 .descriptor[hp_quad_index]
697 .quadrature,
698 update_flags_feval);
699 dealii::FEValues<dim> &fe_val = *fe_values[my_q][fe_index];
700 cell_data.resize(n_q_points);
701
702 // if the FE index has changed from the previous cell, set the
703 // old cell type to invalid (otherwise, we might detect
704 // similarity due to some cells further ahead)
705 if (my_q > 0)
706 cell_t_prev = GeometryType(mapping_info.cell_type[cell]);
707 else if (cell > cell_range.first && active_fe_index.size() > 0 &&
708 active_fe_index[cell] != active_fe_index[cell - 1])
709 cell_t_prev = general;
710
711 evaluate_on_cell(tria,
712 &cells[cell * VectorizedArrayType::size()],
713 my_q,
714 cell_t_prev,
715 cell_t,
716 fe_val,
717 cell_data);
718
719 // now reorder the data into vectorized types. if we are here
720 // for the first time, we need to find out whether the Jacobian
721 // allows for some simplification (Cartesian, affine) taking
722 // vectorization_width cell together
723
724 if (my_q == 0)
725 {
726 // find the most general cell type (most general type is 3
727 // (general cell))
728 GeometryType most_general_type = cartesian;
729 for (unsigned int j = 0; j < VectorizedArrayType::size(); ++j)
730 if (cell_t[j] > most_general_type)
731 most_general_type = cell_t[j];
732 AssertIndexRange(most_general_type, 4U);
733 mapping_info.cell_type[cell] = most_general_type;
734 }
735
736 AssertThrow(
737 data.first[my_q].JxW_values.size() <
738 static_cast<std::size_t>(
739 std::numeric_limits<unsigned int>::max()),
740 ExcMessage(
741 "Index overflow. Cannot fit data in 32 bit integers"));
742
743 unsigned int insert_position = data.first[my_q].JxW_values.size();
744 // Cartesian/affine cell with constant Jacobians throughout the
745 // cell. We need to store the data in another data field because
746 // std::map cannot store data based on VectorizedArray directly
747 // (alignment issue).
748 if (mapping_info.cell_type[cell] <= affine)
749 {
750 if (my_q == 0)
751 {
752 std::pair<
753 Tensor<2,
754 dim,
755 Tensor<1, VectorizedArrayType::size(), Number>>,
756 unsigned int>
757 new_entry;
758 // This number overlaps with the general data but we
759 // take care of that when we merge data from different
760 // threads
761 new_entry.second = data.second.data.size();
762 for (unsigned int d = 0; d < dim; ++d)
763 for (unsigned int e = 0; e < dim; ++e)
764 for (unsigned int v = 0;
765 v < VectorizedArrayType::size();
766 ++v)
767 new_entry.first[d][e][v] =
768 cell_data.const_jac[d][e][v];
769
770 insert_position =
771 data.second.data.insert(new_entry).first->second;
772 }
773 else
774 insert_position =
775 data.first[0].data_index_offsets[cell - cell_range.first];
776 }
777
778 // general cell case: now go through all quadrature points and
779 // collect the data. done for all different quadrature formulas,
780 // so do it outside the above loop.
781 data.first[my_q].data_index_offsets.push_back(insert_position);
782 if (mapping_info.get_cell_type(cell) == general)
783 {
784 for (unsigned int q = 0; q < n_q_points; ++q)
785 {
786 Tensor<2, dim, VectorizedArrayType> &jac =
787 cell_data.general_jac[q];
788 Tensor<3, dim, VectorizedArrayType> &jacobian_grad =
789 cell_data.general_jac_grad[q];
790 for (unsigned int j = 0; j < VectorizedArrayType::size();
791 ++j)
792 if (cell_t[j] < general)
793 {
794 for (unsigned int d = 0; d < dim; ++d)
795 for (unsigned int e = 0; e < dim; ++e)
796 {
797 jac[d][e][j] = cell_data.const_jac[d][e][j];
798 for (unsigned int f = 0; f < dim; ++f)
799 jacobian_grad[d][e][f][j] = 0.;
800 }
801 }
802
803 data.first[my_q].JxW_values.push_back(
804 determinant(jac) * fe_val.get_quadrature().weight(q));
805 Tensor<2, dim, VectorizedArrayType> inv_jac =
806 transpose(invert(jac));
807 data.first[my_q].jacobians[0].push_back(inv_jac);
808
809 if (update_flags & update_jacobian_grads)
810 {
811 data.first[my_q].jacobian_gradients[0].push_back(
812 process_jacobian_gradient(inv_jac,
813 inv_jac,
814 jacobian_grad));
815 Tensor<1,
816 dim *(dim + 1) / 2,
817 Tensor<1, dim, VectorizedArrayType>>
818 jac_grad_sym;
819 // the diagonal part of Jacobian gradient comes
820 // first
821 for (unsigned int d = 0; d < dim; ++d)
822 for (unsigned int f = 0; f < dim; ++f)
823 jac_grad_sym[d][f] = jacobian_grad[f][d][d];
824
825 // then the upper-diagonal part
826 for (unsigned int d = 0, count = dim; d < dim; ++d)
827 for (unsigned int e = d + 1; e < dim; ++e, ++count)
828 for (unsigned int f = 0; f < dim; ++f)
829 jac_grad_sym[count][f] = jacobian_grad[f][d][e];
830
831 data.first[my_q]
832 .jacobian_gradients_non_inverse[0]
833 .push_back(jac_grad_sym);
834 }
835 }
836 }
837
838 if (update_flags & update_quadrature_points)
839 {
840 // eventually we turn to the quadrature points that we can
841 // compress in case we have affine cells. we also need to
842 // reorder them into arrays of vectorized data types. first
843 // go through the cells and find out how much memory we need
844 // to allocate for the quadrature points. We store
845 // n_q_points for general cells and a single value for
846 // Cartesian and affine cells (the position of the (0,0)
847 // point from the reference coordinates)
848 const unsigned int old_size =
849 data.first[my_q].quadrature_points.size();
850 data.first[my_q].quadrature_point_offsets.push_back(old_size);
851
852 if (mapping_info.get_cell_type(cell) < general)
853 {
854 Point<dim, VectorizedArrayType> quad_point;
855 for (unsigned int v = 0; v < VectorizedArrayType::size();
856 ++v)
857 {
858 typename dealii::Triangulation<dim>::cell_iterator
859 cell_it(
860 &tria,
861 cells[cell * VectorizedArrayType::size() + v]
862 .first,
863 cells[cell * VectorizedArrayType::size() + v]
864 .second);
865 const Point<dim> p =
866 mapping[hp_mapping_index]
867 .transform_unit_to_real_cell(cell_it,
868 Point<dim>());
869 for (unsigned int d = 0; d < dim; ++d)
870 quad_point[d][v] = p[d];
871 }
872 data.first[my_q].quadrature_points.push_back(quad_point);
873 }
874 else
875 {
876 for (unsigned int q = 0; q < n_q_points; ++q)
877 data.first[my_q].quadrature_points.push_back(
878 cell_data.quadrature_points[q]);
879 }
880 }
881 } // end for ( cell < end_cells )
882 }
883
884
885
886 template <typename CONTAINER>
887 void
888 merge_compressed_data(const CONTAINER &source,
889 CONTAINER &destination,
890 std::vector<unsigned int> &indices)
891 {
892 indices.resize(source.size());
893 typename CONTAINER::iterator lookup = destination.begin();
894 for (typename CONTAINER::const_iterator it = source.begin();
895 it != source.end();
896 ++it)
897 {
898 typename CONTAINER::value_type entry = *it;
899 entry.second = destination.size();
900 lookup = destination.insert(lookup, entry);
901 AssertIndexRange(it->second, indices.size());
902 indices[it->second] = lookup->second;
903 // best guess for insert position of next item
904 ++lookup;
905 }
906 }
907
908
909
910 template <int structdim,
911 int dim,
912 typename Number,
913 typename VectorizedArrayType>
914 void
915 copy_data(
916 const unsigned int first_cell,
917 const std::array<std::size_t, 2> &data_shift,
918 const std::vector<unsigned int> &indices_compressed,
919 const std::vector<GeometryType> &cell_type,
920 MappingInfoStorage<structdim, dim, VectorizedArrayType>
921 &data_cells_local,
922 MappingInfoStorage<structdim, dim, VectorizedArrayType> &data_cells)
923 {
924 // Copy the index offsets and shift by the appropriate value
925 for (unsigned int lcell = 0;
926 lcell < data_cells_local.data_index_offsets.size();
927 ++lcell)
928 {
929 const unsigned int cell = lcell + first_cell;
930 data_cells.data_index_offsets[cell] =
931 cell_type[cell] <= affine ?
932 (dim == structdim ? 2 : 1) *
933 indices_compressed[data_cells_local
934 .data_index_offsets[lcell]] :
935 data_cells_local.data_index_offsets[lcell] + data_shift[0];
936 if (data_cells_local.quadrature_point_offsets.size() > lcell)
937 data_cells.quadrature_point_offsets[cell] =
938 data_cells_local.quadrature_point_offsets[lcell] +
939 data_shift[1];
940 }
941
942 // Copy quadrature points
943 if (data_cells.quadrature_point_offsets.empty() == false)
944 {
945 Point<dim, VectorizedArrayType> *out_point =
946 &data_cells.quadrature_points[data_shift[1]];
947 for (const Point<dim, VectorizedArrayType> *point =
948 data_cells_local.quadrature_points.begin();
949 point != data_cells_local.quadrature_points.end();
950 ++point, ++out_point)
951 *out_point = *point;
952 data_cells_local.quadrature_points.clear();
953 }
954
955 // If we have collected Jacobian data, copy Jacobians, JxW values,
956 // Jacobian gradients
957 if (data_cells_local.JxW_values.empty())
958 return;
959
960 std::copy(data_cells_local.JxW_values.begin(),
961 data_cells_local.JxW_values.end(),
962 data_cells.JxW_values.begin() + data_shift[0]);
963 data_cells_local.JxW_values.clear();
964 std::copy(data_cells_local.normal_vectors.begin(),
965 data_cells_local.normal_vectors.end(),
966 data_cells.normal_vectors.begin() + data_shift[0]);
967 data_cells_local.normal_vectors.clear();
968 for (unsigned int i = 0; i < 2; ++i)
969 {
970 std::copy(data_cells_local.jacobians[i].begin(),
971 data_cells_local.jacobians[i].end(),
972 data_cells.jacobians[i].begin() + data_shift[0]);
973 data_cells_local.jacobians[i].clear();
974 std::copy(data_cells_local.jacobian_gradients[i].begin(),
975 data_cells_local.jacobian_gradients[i].end(),
976 data_cells.jacobian_gradients[i].begin() + data_shift[0]);
977 data_cells_local.jacobian_gradients[i].clear();
978 std::copy(
979 data_cells_local.jacobian_gradients_non_inverse[i].begin(),
980 data_cells_local.jacobian_gradients_non_inverse[i].end(),
981 data_cells.jacobian_gradients_non_inverse[i].begin() +
982 data_shift[0]);
983 data_cells_local.jacobian_gradients_non_inverse[i].clear();
984 std::copy(data_cells_local.normals_times_jacobians[i].begin(),
985 data_cells_local.normals_times_jacobians[i].end(),
986 data_cells.normals_times_jacobians[i].begin() +
987 data_shift[0]);
988 data_cells_local.normals_times_jacobians[i].clear();
989 }
990 }
991
992
993
994 /**
995 * This function is the preparatory step for the faster MappingQ-based
996 * setup of the data structures in Mapping Info, using FEValues as an
997 * initial query mechanism into MappingQ, storing the resulting
998 * quadrature points and an initial representation of the Jacobians in
999 * two arrays.
1000 */
1001 template <int dim>
1002 void
1003 mapping_q_query_fe_values(
1004 const unsigned int begin_cell,
1005 const unsigned int end_cell,
1006 const MappingQ<dim> &mapping_q,
1007 const dealii::Triangulation<dim> &tria,
1008 const std::vector<std::pair<unsigned int, unsigned int>> &cell_array,
1009 const double jacobian_size,
1010 std::vector<GeometryType> &preliminary_cell_type,
1011 AlignedVector<double> &plain_quadrature_points,
1012 AlignedVector<std::array<Tensor<2, dim>, dim + 1>>
1013 &jacobians_on_stencil)
1014 {
1015 if (begin_cell == end_cell)
1016 return;
1017
1018 const unsigned int mapping_degree = mapping_q.get_degree();
1019 FE_Nothing<dim> dummy_fe;
1020 QGaussLobatto<dim> quadrature(mapping_degree + 1);
1021 const unsigned int n_mapping_points =
1022 Utilities::pow(mapping_degree + 1, dim);
1023
1024 FEValues<dim> fe_values(mapping_q,
1025 dummy_fe,
1026 quadrature,
1027 update_quadrature_points | update_jacobians);
1028
1029 for (unsigned int cell = begin_cell; cell < end_cell; ++cell)
1030 {
1031 typename dealii::Triangulation<dim>::cell_iterator cell_it(
1032 &tria, cell_array[cell].first, cell_array[cell].second);
1033 fe_values.reinit(cell_it);
1034 for (unsigned int d = 0; d < dim; ++d)
1035 for (unsigned int q = 0; q < n_mapping_points; ++q)
1036 plain_quadrature_points[(cell * dim + d) * n_mapping_points +
1037 q] = fe_values.quadrature_point(q)[d];
1038
1039 // store the first, second, n-th and n^2-th one along a
1040 // stencil-like pattern
1041 std::array<Tensor<2, dim, double>, dim + 1> &my_jacobians =
1042 jacobians_on_stencil[cell];
1043 my_jacobians[0] = Tensor<2, dim, double>(fe_values.jacobian(0));
1044 for (unsigned int d = 0, skip = 1; d < dim;
1045 ++d, skip *= (mapping_degree + 1))
1046 my_jacobians[1 + d] =
1047 Tensor<2, dim, double>(fe_values.jacobian(skip));
1048
1049 // check whether cell is Cartesian/affine/general
1050 GeometryType type = cartesian;
1051 for (unsigned int d = 0; d < dim; ++d)
1052 for (unsigned int e = 0; e < dim; ++e)
1053 if (d != e)
1054 if (std::abs(my_jacobians[0][d][e]) > 1e-12 * jacobian_size)
1055 type = affine;
1056
1057 for (unsigned int q = 1; q < n_mapping_points; ++q)
1058 for (unsigned int d = 0; d < dim; ++d)
1059 for (unsigned int e = 0; e < dim; ++e)
1060 if (std::abs(fe_values.jacobian(q)[d][e] -
1061 fe_values.jacobian(0)[d][e]) >
1062 1e-12 * jacobian_size)
1063 {
1064 type = general;
1065 goto endloop;
1066 }
1067 endloop:
1068 preliminary_cell_type[cell] = type;
1069 }
1070 }
1071
1072
1073
1074 template <int dim>
1075 std::vector<unsigned int>
1076 mapping_q_find_compression(
1077 const double jacobian_size,
1078 const AlignedVector<std::array<Tensor<2, dim>, dim + 1>>
1079 &jacobians_on_stencil,
1080 const unsigned int n_mapping_points,
1081 const AlignedVector<double> &plain_quadrature_points,
1082 std::vector<GeometryType> &preliminary_cell_type)
1083 {
1084 std::vector<unsigned int> cell_data_index(jacobians_on_stencil.size());
1085
1086 // we include a map to store some compressed information about the
1087 // Jacobians which we collect by a stencil-like pattern around the
1088 // first quadrature point on the cell - we use a relatively coarse
1089 // tolerance to account for some inaccuracies in the manifold
1090 // evaluation
1091 std::map<std::array<Tensor<2, dim>, dim + 1>,
1092 unsigned int,
1093 FloatingPointComparator<double>>
1094 compressed_jacobians(FloatingPointComparator<double>(
1095 1e4 * jacobian_size * std::numeric_limits<double>::epsilon() *
1096 1024.));
1097
1098 unsigned int n_data_buckets = 0;
1099 for (unsigned int cell = 0; cell < jacobians_on_stencil.size(); ++cell)
1100 {
1101 // check in the map for the index of this cell
1102 const auto position =
1103 compressed_jacobians.find(jacobians_on_stencil[cell]);
1104 bool add_this_cell = position == compressed_jacobians.end();
1105 if (!add_this_cell)
1106 {
1107 // check if the found duplicate really is a translation and
1108 // the similarity identified by the map is not by accident
1109 double max_distance = 0;
1110 const double *ptr_origin =
1111 plain_quadrature_points.data() +
1112 position->second * dim * n_mapping_points;
1113 const double *ptr_mine = plain_quadrature_points.data() +
1114 cell * dim * n_mapping_points;
1115 for (unsigned int d = 0; d < dim; ++d)
1116 {
1117 const double translate_d =
1118 ptr_origin[d * n_mapping_points] -
1119 ptr_mine[d * n_mapping_points];
1120 for (unsigned int q = 1; q < n_mapping_points; ++q)
1121 max_distance =
1122 std::max(std::abs(ptr_origin[d * n_mapping_points + q] -
1123 ptr_mine[d * n_mapping_points + q] -
1124 translate_d),
1125 max_distance);
1126 }
1127
1128 // this is not a duplicate, must add it again
1129 if (max_distance > 1e-10 * jacobian_size)
1130 add_this_cell = true;
1131 }
1132 else
1133 compressed_jacobians.insert(
1134 std::make_pair(jacobians_on_stencil[cell], cell));
1135
1136 if (add_this_cell)
1137 cell_data_index[cell] = n_data_buckets++;
1138 else
1139 {
1140 cell_data_index[cell] = cell_data_index[position->second];
1141 // make sure that the cell type is the same as in the original
1142 // field, despite possibly small differences due to roundoff
1143 // and the tolerances we use
1144 preliminary_cell_type[cell] =
1145 preliminary_cell_type[position->second];
1146 }
1147 }
1148 return cell_data_index;
1149 }
1150
1151
1152 /**
1153 * This function computes and tabulates the mapping information for
1154 * MappingQ-derived mappings on a range of cells calling into the tensor
1155 * product evaluators of the matrix-free framework, using a
1156 * polynomial expansion of the cell geometry underlying the MappingQ
1157 * class.
1158 */
1159 template <int dim,
1160 typename Number,
1161 typename VectorizedArrayType,
1162 typename VectorizedDouble>
1163 void
1164 mapping_q_compute_range(
1165 const unsigned int begin_cell,
1166 const unsigned int end_cell,
1167 const dealii::Triangulation<dim> &tria,
1168 const std::vector<std::pair<unsigned int, unsigned int>> &cell_array,
1169 const std::vector<GeometryType> &cell_type,
1170 const std::vector<bool> &process_cell,
1171 const UpdateFlags update_flags_cells,
1172 const AlignedVector<double> &plain_quadrature_points,
1173 const ShapeInfo<double> &shape_info,
1174 MappingInfoStorage<dim, dim, VectorizedArrayType> &my_data)
1175 {
1176 constexpr unsigned int n_lanes = VectorizedArrayType::size();
1177 constexpr unsigned int n_lanes_d = VectorizedDouble::size();
1178
1179 const unsigned int n_q_points = my_data.descriptor[0].n_q_points;
1180 const unsigned int n_mapping_points =
1181 shape_info.dofs_per_component_on_cell;
1182 constexpr unsigned int hess_dim = dim * (dim + 1) / 2;
1183
1184 FEEvaluationData<dim, VectorizedDouble, false> eval(shape_info);
1185
1186 AlignedVector<VectorizedDouble> evaluation_data;
1187 eval.set_data_pointers(&evaluation_data, dim);
1188
1189 for (unsigned int cell = begin_cell; cell < end_cell; ++cell)
1190 for (unsigned vv = 0; vv < n_lanes; vv += n_lanes_d)
1191 {
1192 if (cell_type[cell] > affine || process_cell[cell])
1193 {
1194 unsigned int start_indices[n_lanes_d];
1195 for (unsigned int v = 0; v < n_lanes_d; ++v)
1196 start_indices[v] =
1197 (cell * n_lanes + vv + v) * n_mapping_points * dim;
1198 vectorized_load_and_transpose(n_mapping_points * dim,
1199 plain_quadrature_points.data(),
1200 start_indices,
1201 eval.begin_dof_values());
1202
1203 FEEvaluationFactory<dim, VectorizedDouble>::evaluate(
1204 dim,
1205 EvaluationFlags::values | EvaluationFlags::gradients |
1206 (update_flags_cells & update_jacobian_grads ?
1207 EvaluationFlags::hessians :
1208 EvaluationFlags::nothing),
1209 eval.begin_dof_values(),
1210 eval);
1211 }
1212 if (update_flags_cells & update_quadrature_points)
1213 {
1214 Point<dim, VectorizedArrayType> *quadrature_points =
1215 my_data.quadrature_points.data() +
1216 my_data.quadrature_point_offsets[cell];
1217 if (cell_type[cell] <= affine)
1218 for (unsigned int d = 0; d < dim; ++d)
1219 for (unsigned int v = 0; v < n_lanes_d; ++v)
1220 quadrature_points[0][d][vv + v] =
1221 plain_quadrature_points
1222 [(dim * (cell * n_lanes + vv + v) + d) *
1223 n_mapping_points];
1224 else
1225 for (unsigned int d = 0; d < dim; ++d)
1226 for (unsigned int q = 0; q < n_q_points; ++q)
1227 store_vectorized_array(
1228 eval.begin_values()[q + d * n_q_points],
1229 vv,
1230 quadrature_points[q][d]);
1231 }
1232
1233 const unsigned int n_points =
1234 cell_type[cell] <= affine ? 1 : n_q_points;
1235 if (process_cell[cell])
1236 for (unsigned int q = 0; q < n_points; ++q)
1237 {
1238 const unsigned int idx =
1239 my_data.data_index_offsets[cell] + q;
1240 Tensor<2, dim, VectorizedDouble> jac;
1241 for (unsigned int d = 0; d < dim; ++d)
1242 for (unsigned int e = 0; e < dim; ++e)
1243 jac[d][e] =
1244 eval
1245 .begin_gradients()[e + (d * n_q_points + q) * dim];
1246
1247 // eliminate roundoff errors
1248 if (cell_type[cell] == cartesian)
1249 for (unsigned int d = 0; d < dim; ++d)
1250 for (unsigned int e = 0; e < dim; ++e)
1251 if (d != e)
1252 jac[d][e] = 0.;
1253
1254 const VectorizedDouble jac_det = determinant(jac);
1255
1256 if constexpr (running_in_debug_mode())
1257 {
1258 for (unsigned int v = 0; v < n_lanes_d; ++v)
1259 {
1260 const typename Triangulation<dim>::cell_iterator
1261 cell_iterator(
1262 &tria,
1263 cell_array[cell * n_lanes + vv + v].first,
1264 cell_array[cell * n_lanes + vv + v].second);
1265
1266 Assert(
1267 jac_det[v] > 1e-12 * Utilities::fixed_power<dim>(
1268 cell_iterator->diameter() /
1269 std::sqrt(double(dim))),
1270 (typename Mapping<dim>::ExcDistortedMappedCell(
1271 cell_iterator->center(), jac_det[v], q)));
1272 }
1273 }
1274 else
1275 {
1276 (void)tria;
1277 (void)cell_array;
1278 }
1279
1280 const Tensor<2, dim, VectorizedDouble> inv_jac =
1281 transpose(invert(jac));
1282
1283 if (cell_type[cell] <= affine)
1284 {
1285 store_vectorized_array(jac_det,
1286 vv,
1287 my_data.JxW_values[idx]);
1288
1289 for (unsigned int d = 0; d < dim; ++d)
1290 for (unsigned int e = 0; e < dim; ++e)
1291 store_vectorized_array(
1292 jac[d][e],
1293 vv,
1294 my_data.jacobians[0][idx + 1][d][e]);
1295 }
1296 else
1297 {
1298 const double weight =
1299 my_data.descriptor[0].quadrature.weight(q);
1300 store_vectorized_array(jac_det * weight,
1301 vv,
1302 my_data.JxW_values[idx]);
1303 }
1304 for (unsigned int d = 0; d < dim; ++d)
1305 for (unsigned int e = 0; e < dim; ++e)
1306 store_vectorized_array(inv_jac[d][e],
1307 vv,
1308 my_data.jacobians[0][idx][d][e]);
1309
1310 if (update_flags_cells & update_jacobian_grads &&
1311 cell_type[cell] > affine)
1312 {
1313 Tensor<3, dim, VectorizedDouble> jac_grad;
1314 for (unsigned int d = 0; d < dim; ++d)
1315 {
1316 for (unsigned int e = 0; e < dim; ++e)
1317 jac_grad[d][e][e] =
1318 eval.begin_hessians()[q + (d * hess_dim + e) *
1319 n_q_points];
1320 for (unsigned int c = dim, e = 0; e < dim; ++e)
1321 for (unsigned int f = e + 1; f < dim; ++f, ++c)
1322 jac_grad[d][e][f] = jac_grad[d][f][e] =
1323 eval.begin_hessians()[q + (d * hess_dim + c) *
1324 n_q_points];
1325 const auto inv_jac_grad =
1326 process_jacobian_gradient(inv_jac,
1327 inv_jac,
1328 jac_grad);
1329 for (unsigned int d = 0; d < hess_dim; ++d)
1330 for (unsigned int e = 0; e < dim; ++e)
1331 store_vectorized_array(
1332 inv_jac_grad[d][e],
1333 vv,
1334 my_data.jacobian_gradients[0][idx][d][e]);
1335
1336 // Also store the non-inverse Jacobian gradient.
1337 // the diagonal part of Jacobian gradient comes
1338 // first
1339 for (unsigned int d = 0; d < dim; ++d)
1340 for (unsigned int f = 0; f < dim; ++f)
1341 store_vectorized_array(
1342 jac_grad[f][d][d],
1343 vv,
1344 my_data.jacobian_gradients_non_inverse[0][idx]
1345 [d][f]);
1346
1347 // then the upper-diagonal part
1348 for (unsigned int d = 0, count = dim; d < dim; ++d)
1349 for (unsigned int e = d + 1; e < dim;
1350 ++e, ++count)
1351 for (unsigned int f = 0; f < dim; ++f)
1352 store_vectorized_array(
1353 jac_grad[f][d][e],
1354 vv,
1355 my_data.jacobian_gradients_non_inverse
1356 [0][idx][count][f]);
1357 }
1358 }
1359 }
1360 }
1361 }
1362
1363 } // namespace ExtractCellHelper
1364
1365
1366
1367 template <int dim, typename Number, typename VectorizedArrayType>
1368 void
1369 MappingInfo<dim, Number, VectorizedArrayType>::initialize_cells(
1370 const dealii::Triangulation<dim> &tria,
1371 const std::vector<std::pair<unsigned int, unsigned int>> &cells,
1372 const std::vector<unsigned int> &active_fe_index,
1373 const dealii::hp::MappingCollection<dim> &mapping)
1374 {
1375 const unsigned int n_cells = cells.size();
1376 const unsigned int n_lanes = VectorizedArrayType::size();
1377 Assert(n_cells % n_lanes == 0, ExcInternalError());
1378 const unsigned int n_cell_batches = n_cells / n_lanes;
1379 cell_type.resize(n_cell_batches);
1380
1381 if (n_cell_batches == 0)
1382 return;
1383
1384 // Create as many chunks of cells as we have threads and spawn the work
1385 unsigned int work_per_chunk =
1386 std::max(8U,
1387 (n_cell_batches + MultithreadInfo::n_threads() - 1) /
1388 MultithreadInfo::n_threads());
1389
1390 std::vector<std::pair<
1391 std::vector<MappingInfoStorage<dim, dim, VectorizedArrayType>>,
1392 ExtractCellHelper::
1393 CompressedCellData<dim, Number, VectorizedArrayType>>>
1394 data_cells_local;
1395 // Reserve enough space to avoid re-allocation (which would break the
1396 // references to the data fields passed to the tasks!)
1397 data_cells_local.reserve(MultithreadInfo::n_threads());
1398
1399 {
1400 Threads::TaskGroup<> tasks;
1401 std::pair<unsigned int, unsigned int> cell_range(0U, work_per_chunk);
1402 while (cell_range.first < n_cell_batches)
1403 {
1404 data_cells_local.push_back(std::make_pair(
1405 std::vector<MappingInfoStorage<dim, dim, VectorizedArrayType>>(
1406 cell_data.size()),
1407 ExtractCellHelper::
1408 CompressedCellData<dim, Number, VectorizedArrayType>(
1409 ExtractCellHelper::get_jacobian_size(tria))));
1410 tasks += Threads::new_task(
1411 &ExtractCellHelper::
1412 initialize_cell_range<dim, Number, VectorizedArrayType>,
1413 cell_range,
1414 tria,
1415 cells,
1416 active_fe_index,
1417 mapping,
1418 *this,
1419 data_cells_local.back());
1420 cell_range.first = cell_range.second;
1421 cell_range.second += work_per_chunk;
1422 }
1423 tasks.join_all();
1424 }
1425
1426 // Fill in each thread's constant Jacobians into the data of the zeroth
1427 // chunk in serial
1428 std::vector<std::vector<unsigned int>> indices_compressed(
1429 data_cells_local.size());
1430 for (unsigned int i = 0; i < data_cells_local.size(); ++i)
1431 ExtractCellHelper::merge_compressed_data(
1432 data_cells_local[i].second.data,
1433 data_cells_local[0].second.data,
1434 indices_compressed[i]);
1435
1436 // Collect all data in the final data fields.
1437 // First allocate the memory
1438 const unsigned int n_constant_jacobians =
1439 data_cells_local[0].second.data.size();
1440 for (unsigned int my_q = 0; my_q < cell_data.size(); ++my_q)
1441 {
1442 cell_data[my_q].data_index_offsets.resize(cell_type.size());
1443 std::vector<std::array<std::size_t, 2>> shift(
1444 data_cells_local.size());
1445 shift[0][0] = 2 * n_constant_jacobians;
1446 shift[0][1] = 0;
1447 for (unsigned int i = 1; i < data_cells_local.size(); ++i)
1448 {
1449 shift[i][0] =
1450 shift[i - 1][0] +
1451 data_cells_local[i - 1].first[my_q].JxW_values.size();
1452 shift[i][1] =
1453 shift[i - 1][1] +
1454 data_cells_local[i - 1].first[my_q].quadrature_points.size();
1455 }
1456 cell_data[my_q].JxW_values.resize_fast(
1457 shift.back()[0] +
1458 data_cells_local.back().first[my_q].JxW_values.size());
1459 cell_data[my_q].jacobians[0].resize_fast(
1460 cell_data[my_q].JxW_values.size());
1461 if (update_flags_cells & update_jacobian_grads)
1462 {
1463 cell_data[my_q].jacobian_gradients[0].resize_fast(
1464 cell_data[my_q].JxW_values.size());
1465 cell_data[my_q].jacobian_gradients_non_inverse[0].resize_fast(
1466 cell_data[my_q].JxW_values.size());
1467 }
1468 if (update_flags_cells & update_quadrature_points)
1469 {
1470 cell_data[my_q].quadrature_point_offsets.resize(cell_type.size());
1471 cell_data[my_q].quadrature_points.resize_fast(
1472 shift.back()[1] +
1473 data_cells_local.back().first[my_q].quadrature_points.size());
1474 }
1475
1476 // Start tasks that copy the local data
1477 Threads::TaskGroup<> tasks;
1478 for (unsigned int i = 0; i < data_cells_local.size(); ++i)
1479 tasks += Threads::new_task(
1480 &ExtractCellHelper::
1481 copy_data<dim, dim, Number, VectorizedArrayType>,
1482 work_per_chunk * i,
1483 shift[i],
1484 indices_compressed[i],
1485 cell_type,
1486 data_cells_local[i].first[my_q],
1487 cell_data[my_q]);
1488
1489 // finally, insert the constant cell data at the beginning (the
1490 // other tasks can already start copying the non-constant
1491 // data). Note that we use two slots for the constant data to
1492 // accommodate for both the inverse transposed Jacobian (that we
1493 // need for derivatives) and the Jacobian (that we need for
1494 // quadrature points)
1495 if (my_q == 0)
1496 {
1497 for (const auto &it : data_cells_local[0].second.data)
1498 {
1499 Tensor<2, dim, VectorizedArrayType> jac;
1500 for (unsigned int d = 0; d < dim; ++d)
1501 for (unsigned int e = 0; e < dim; ++e)
1502 for (unsigned int v = 0; v < VectorizedArrayType::size();
1503 ++v)
1504 jac[d][e][v] = it.first[d][e][v];
1505 AssertIndexRange(it.second, n_constant_jacobians);
1506 const std::size_t index = it.second;
1507 cell_data[my_q].JxW_values[2 * index] = determinant(jac);
1508 // invert and transpose jac
1509 cell_data[my_q].jacobians[0][2 * index] =
1510 transpose(invert(jac));
1511 cell_data[my_q].jacobians[0][2 * index + 1] = jac;
1512 // second derivative of transformation is zero on affine cells
1513 }
1514 }
1515 else
1516 {
1517 for (unsigned int i = 0; i < 2 * n_constant_jacobians; ++i)
1518 {
1519 cell_data[my_q].JxW_values[i] = cell_data[0].JxW_values[i];
1520 cell_data[my_q].jacobians[0][i] =
1521 cell_data[0].jacobians[0][i];
1522 }
1523 }
1524
1525 // ... wait for the parallel work to finish
1526 tasks.join_all();
1527 }
1528 }
1529
1530
1531
1532 /* ------------------------- initialization of faces ------------------- */
1533
1534 // Namespace with implementation of extraction of values on face
1535 // range
1536 namespace ExtractFaceHelper
1537 {
1538 template <int dim, typename Number, typename VectorizedArrayType>
1539 struct CompressedFaceData
1540 {
1541 // Constructor. As a scaling factor for the FloatingPointComparator, we
1542 // select the inverse of the Jacobian (not the Jacobian as in the
1543 // CompressedCellData) and add another factor of 512 to account for
1544 // some roundoff effects.
1545 CompressedFaceData(const Number jacobian_size)
1546 : data(FloatingPointComparator<VectorizedArrayType>(
1547 512. / jacobian_size * std::numeric_limits<double>::epsilon() *
1548 1024.))
1549 , jacobian_size(jacobian_size)
1550 {}
1551
1552 // Store the Jacobians on both sides of a face (2*(dim*dim) entries),
1553 // the normal vector (dim entries), and the Jacobian determinant on
1554 // the face (first tensor) for each entry in the vectorized array
1555 // (inner tensor). We cannot choose a VectorizedArray type directly
1556 // because std::map does not provide the necessary alignment upon
1557 // memory allocation.
1558 std::map<Tensor<1,
1559 2 * dim * dim + dim + 1,
1560 Tensor<1, VectorizedArrayType::size(), Number>>,
1561 unsigned int,
1562 FloatingPointComparator<VectorizedArrayType>>
1563 data;
1564
1565 // Store the scaling factor
1566 const Number jacobian_size;
1567 };
1568
1569
1570
1571 // We always put the derivative normal to the face in the last slot for
1572 // simpler unit cell gradient computations. This function reorders the
1573 // indices of the Jacobian appropriately.
1574 template <int dim>
1575 unsigned int
1576 reorder_face_derivative_indices(
1577 const unsigned int face_no,
1578 const unsigned int index,
1579 const ReferenceCell reference_cell = ReferenceCells::Invalid)
1580 {
1581 Assert(index < dim, ExcInternalError());
1582
1583 if ((reference_cell == ReferenceCells::Invalid ||
1584 reference_cell == ReferenceCells::get_hypercube<dim>()) == false)
1585 {
1586 return index;
1587 }
1588
1589 if (dim == 3)
1590 {
1591 unsigned int table[3][3] = {{1, 2, 0}, {2, 0, 1}, {0, 1, 2}};
1592 return table[face_no / 2][index];
1593 }
1594 else if (dim == 2)
1595 {
1596 unsigned int table[2][2] = {{1, 0}, {0, 1}};
1597 return table[face_no / 2][index];
1598 }
1599 else if (dim == 1)
1600 return 0;
1601 else
1602 Assert(false,
1603 ExcNotImplemented("Not possible in dim=" +
1604 std::to_string(dim)));
1605
1606 return numbers::invalid_unsigned_int;
1607 }
1608
1609
1610
1611 /**
1612 * This function computes and tabulates the mapping information for
1613 * generic mappings on a range of faces by a call to FEFaceValues with
1614 * the respective UpdateFlags set. There is a specialized function
1615 * mapping_q_compute_range function that provides a faster
1616 * initialization for mappings derived from MappingQ and suitable
1617 * quadrature formulas.
1618 */
1619 template <int dim, typename Number, typename VectorizedArrayType>
1620 void
1621 initialize_face_range(
1622 const std::pair<unsigned int, unsigned int> face_range,
1623 const dealii::Triangulation<dim> &tria,
1624 const std::vector<std::pair<unsigned int, unsigned int>> &cells,
1625 const std::vector<FaceToCellTopology<VectorizedArrayType::size()>>
1626 &faces,
1627 const std::vector<unsigned int> &active_fe_index,
1628 const dealii::hp::MappingCollection<dim> &mapping_in,
1629 MappingInfo<dim, Number, VectorizedArrayType> &mapping_info,
1630 std::pair<
1631 std::vector<MappingInfoStorage<dim - 1, dim, VectorizedArrayType>>,
1632 CompressedFaceData<dim, Number, VectorizedArrayType>> &data)
1633 {
1634 std::vector<std::vector<std::shared_ptr<FE_Nothing<dim>>>> dummy_fe(
1635 mapping_info.reference_cell_types.size());
1636 for (unsigned int my_q = 0;
1637 my_q < mapping_info.reference_cell_types.size();
1638 ++my_q)
1639 {
1640 const unsigned int n_hp_quads =
1641 mapping_info.reference_cell_types[my_q].size();
1642 dummy_fe[my_q].resize(n_hp_quads);
1643
1644 for (unsigned int hpq = 0; hpq < n_hp_quads; ++hpq)
1645 dummy_fe[my_q][hpq] = std::make_shared<FE_Nothing<dim>>(
1646 mapping_info.reference_cell_types[my_q][hpq]);
1647 }
1648
1649 const unsigned int max_active_fe_index =
1650 active_fe_index.size() > 0 ?
1651 *std::max_element(active_fe_index.begin(), active_fe_index.end()) :
1652 0;
1653
1654 std::vector<std::vector<std::shared_ptr<FEFaceValues<dim>>>>
1655 fe_face_values_container(mapping_info.face_data.size());
1656 for (unsigned int my_q = 0; my_q < mapping_info.face_data.size();
1657 ++my_q)
1658 fe_face_values_container[my_q].resize(max_active_fe_index + 1);
1659
1660 std::vector<std::vector<std::shared_ptr<FEFaceValues<dim>>>>
1661 fe_boundary_face_values_container(mapping_info.face_data.size());
1662 for (unsigned int my_q = 0; my_q < mapping_info.face_data.size();
1663 ++my_q)
1664 fe_boundary_face_values_container[my_q].resize(max_active_fe_index +
1665 1);
1666
1667 std::vector<std::vector<std::shared_ptr<FESubfaceValues<dim>>>>
1668 fe_subface_values_container(mapping_info.face_data.size());
1669 for (unsigned int my_q = 0; my_q < mapping_info.face_data.size();
1670 ++my_q)
1671 fe_subface_values_container[my_q].resize(
1672 mapping_info.face_data[my_q].descriptor.size());
1673
1674 ExtractCellHelper::LocalData<dim, Number, VectorizedArrayType>
1675 face_data(ExtractCellHelper::get_jacobian_size(tria));
1676
1677 const unsigned int end_face =
1678 std::min(std::size_t(face_range.second), faces.size());
1679 for (unsigned int face = face_range.first; face < end_face; ++face)
1680 for (unsigned int my_q = 0; my_q < mapping_info.face_data.size();
1681 ++my_q)
1682 {
1683 // We assume that we have the faces sorted by the active FE
1684 // indices so that the active FE index of the interior side of the
1685 // face batch is the same as the FE index of the interior side of
1686 // its first entry.
1687 const unsigned int fe_index =
1688 active_fe_index.size() > 0 ?
1689 active_fe_index[faces[face].cells_interior[0] /
1690 VectorizedArrayType::size()] :
1691 0;
1692 const unsigned int hp_quad_index =
1693 mapping_info.cell_data[my_q].descriptor.size() == 1 ? 0 :
1694 fe_index;
1695 const unsigned int hp_mapping_index =
1696 mapping_in.size() == 1 ? 0 : fe_index;
1697
1698 const auto &mapping = mapping_in[hp_mapping_index];
1699 const auto &quadrature =
1700 mapping_info.face_data[my_q].q_collection[hp_quad_index];
1701
1702 const bool is_boundary_face =
1703 faces[face].cells_exterior[0] == numbers::invalid_unsigned_int;
1704
1705 if (is_boundary_face &&
1706 fe_boundary_face_values_container[my_q][fe_index] == nullptr)
1707 fe_boundary_face_values_container[my_q][fe_index] =
1708 std::make_shared<FEFaceValues<dim>>(
1709 mapping,
1710 *dummy_fe[my_q][hp_quad_index],
1711 quadrature,
1712 mapping_info.update_flags_boundary_faces);
1713 else if (fe_face_values_container[my_q][fe_index] == nullptr)
1714 fe_face_values_container[my_q][fe_index] =
1715 std::make_shared<FEFaceValues<dim>>(
1716 mapping,
1717 *dummy_fe[my_q][hp_quad_index],
1718 quadrature,
1719 mapping_info.update_flags_inner_faces);
1720
1721 FEFaceValues<dim> &fe_face_values =
1722 is_boundary_face ?
1723 *fe_boundary_face_values_container[my_q][fe_index] :
1724 *fe_face_values_container[my_q][fe_index];
1725
1726 unsigned int n_q_points = 0; // will be override once FEFaceValues
1727 // is set up
1728
1729 bool normal_is_similar = true;
1730 bool JxW_is_similar = true;
1731 bool cell_is_cartesian = true;
1732 for (unsigned int v = 0; v < VectorizedArrayType::size(); ++v)
1733 {
1734 Tensor<2, dim> jacobian_0;
1735 double compare_norm_jac = 1.;
1736 if (faces[face].cells_interior[v] !=
1737 numbers::invalid_unsigned_int)
1738 {
1739 typename dealii::Triangulation<dim>::cell_iterator
1740 cell_it(&tria,
1741 cells[faces[face].cells_interior[v]].first,
1742 cells[faces[face].cells_interior[v]].second);
1743
1744 fe_face_values.reinit(cell_it,
1745 faces[face].interior_face_no);
1746
1747 if (v == 0)
1748 {
1749 n_q_points = fe_face_values.n_quadrature_points;
1750 face_data.resize(n_q_points);
1751 }
1752
1753 for (unsigned int q = 0; q < n_q_points; ++q)
1754 {
1755 if (std::abs(
1756 fe_face_values.JxW(q) *
1757 fe_face_values.get_quadrature().weight(0) -
1758 fe_face_values.JxW(0) *
1759 fe_face_values.get_quadrature().weight(q)) >
1760 2048. * std::numeric_limits<double>::epsilon() *
1761 fe_face_values.JxW(0) *
1762 fe_face_values.get_quadrature().weight(q))
1763 JxW_is_similar = false;
1764 face_data.JxW_values[q][v] = fe_face_values.JxW(q);
1765
1766 DerivativeForm<1, dim, dim> inv_jac =
1767 fe_face_values.jacobian(q).covariant_form();
1768 Tensor<1, dim> normal =
1769 fe_face_values.normal_vector(q);
1770
1771 // Filter out very small values in normal. No need
1772 // to re-normalize because these values cannot enter
1773 // the norm significantly: Total size is 1 but 1e-13
1774 // squared is 1e-26.
1775 for (unsigned int d = 0; d < dim; ++d)
1776 if (std::abs(normal[d]) <
1777 1024. * std::numeric_limits<double>::epsilon())
1778 normal[d] = 0.;
1779
1780 if (q == 0)
1781 {
1782 jacobian_0 = inv_jac;
1783 compare_norm_jac = jacobian_0.norm();
1784 }
1785 for (unsigned int d = 0; d < dim; ++d)
1786 for (unsigned int e = 0; e < dim; ++e)
1787 {
1788 if (std::abs(inv_jac[d][e] - jacobian_0[d][e]) >
1789 2048. *
1790 std::numeric_limits<double>::epsilon() *
1791 compare_norm_jac)
1792 JxW_is_similar = false;
1793 const unsigned int ee =
1794 reorder_face_derivative_indices<dim>(
1795 faces[face].interior_face_no,
1796 e,
1797 cell_it->reference_cell());
1798 face_data.general_jac[q][d][e][v] =
1799 inv_jac[d][ee];
1800 }
1801
1802 for (unsigned int d = 0; d < dim; ++d)
1803 {
1804 face_data.normal_vectors[q][d][v] = normal[d];
1805 if (std::abs(normal[d] -
1806 fe_face_values.normal_vector(0)[d]) >
1807 1024. *
1808 std::numeric_limits<double>::epsilon())
1809 normal_is_similar = false;
1810 }
1811 if (std::abs(
1812 std::abs(
1813 normal[faces[face].interior_face_no / 2]) -
1814 1.) >
1815 1024. * std::numeric_limits<double>::epsilon())
1816 cell_is_cartesian = false;
1817 for (unsigned int d = 0; d < dim; ++d)
1818 for (unsigned int e = 0; e < dim; ++e)
1819 if (e != d &&
1820 std::abs(inv_jac[d][e]) >
1821 2048 *
1822 std::numeric_limits<double>::epsilon() *
1823 compare_norm_jac)
1824 cell_is_cartesian = false;
1825
1826 if (fe_face_values.get_update_flags() &
1827 update_quadrature_points)
1828 for (unsigned int d = 0; d < dim; ++d)
1829 face_data.quadrature_points[q][d][v] =
1830 fe_face_values.quadrature_point(q)[d];
1831 }
1832 }
1833 // Fill up with data of the zeroth component to avoid
1834 // false negatives when checking for similarities
1835 else
1836 {
1837 for (unsigned int q = 0; q < n_q_points; ++q)
1838 {
1839 face_data.JxW_values[q][v] =
1840 face_data.JxW_values[q][0];
1841 for (unsigned int d = 0; d < dim; ++d)
1842 for (unsigned int e = 0; e < dim; ++e)
1843 {
1844 face_data.general_jac[q][d][e][v] =
1845 face_data.general_jac[q][d][e][0];
1846 face_data.normal_vectors[q][d][v] =
1847 face_data.normal_vectors[q][d][0];
1848 }
1849 if (fe_face_values.get_update_flags() &
1850 update_quadrature_points)
1851 for (unsigned int d = 0; d < dim; ++d)
1852 face_data.quadrature_points[q][d][v] =
1853 face_data.quadrature_points[q][d][0];
1854 }
1855 }
1856 if (is_boundary_face == false &&
1857 faces[face].cells_exterior[v] !=
1858 numbers::invalid_unsigned_int)
1859 {
1860 typename dealii::Triangulation<dim>::cell_iterator
1861 cell_it(&tria,
1862 cells[faces[face].cells_exterior[v]].first,
1863 cells[faces[face].cells_exterior[v]].second);
1864
1865 const FEValuesBase<dim> *actual_fe_face_values = nullptr;
1866 if (faces[face].subface_index >=
1867 GeometryInfo<dim>::max_children_per_cell)
1868 {
1869 // We assume that we have the faces sorted by the
1870 // active FE indices so that the active FE index of
1871 // the exterior side of the face batch is the same as
1872 // the FE index of the exterior side of its first
1873 // entry.
1874 const unsigned int fe_index =
1875 active_fe_index.size() > 0 ?
1876 active_fe_index[faces[face].cells_exterior[0] /
1877 VectorizedArrayType::size()] :
1878 0;
1879 const unsigned int hp_quad_index =
1880 mapping_info.cell_data[my_q].descriptor.size() ==
1881 1 ?
1882 0 :
1883 fe_index;
1884 const unsigned int hp_mapping_index =
1885 mapping_in.size() == 1 ? 0 : fe_index;
1886
1887 const auto &mapping = mapping_in[hp_mapping_index];
1888 const auto &quadrature =
1889 mapping_info.face_data[my_q]
1890 .q_collection[hp_quad_index];
1891
1892 if (fe_face_values_container[my_q][fe_index] ==
1893 nullptr)
1894 fe_face_values_container[my_q][fe_index] =
1895 std::make_shared<FEFaceValues<dim>>(
1896 mapping,
1897 *dummy_fe[my_q][hp_quad_index],
1898 quadrature,
1899 mapping_info.update_flags_boundary_faces);
1900
1901 fe_face_values_container[my_q][fe_index]->reinit(
1902 cell_it, faces[face].exterior_face_no);
1903
1904 actual_fe_face_values =
1905 fe_face_values_container[my_q][fe_index].get();
1906 }
1907 else
1908 {
1909 if (fe_subface_values_container[my_q][0] == nullptr)
1910 fe_subface_values_container[my_q][0] =
1911 std::make_shared<FESubfaceValues<dim>>(
1912 mapping,
1913 *dummy_fe[my_q][hp_quad_index],
1914 quadrature,
1915 mapping_info.update_flags_inner_faces);
1916 fe_subface_values_container[my_q][0]->reinit(
1917 cell_it,
1918 faces[face].exterior_face_no,
1919 faces[face].subface_index);
1920 actual_fe_face_values =
1921 fe_subface_values_container[my_q][0].get();
1922 }
1923 for (unsigned int q = 0; q < n_q_points; ++q)
1924 {
1925 DerivativeForm<1, dim, dim> inv_jac =
1926 actual_fe_face_values->jacobian(q).covariant_form();
1927 if (q == 0)
1928 {
1929 jacobian_0 = inv_jac;
1930 compare_norm_jac = jacobian_0.norm();
1931 }
1932 for (unsigned int d = 0; d < dim; ++d)
1933 for (unsigned int e = 0; e < dim; ++e)
1934 {
1935 if (std::abs(inv_jac[d][e] - jacobian_0[d][e]) >
1936 2048. *
1937 std::numeric_limits<double>::epsilon() *
1938 compare_norm_jac)
1939 JxW_is_similar = false;
1940 const unsigned int ee =
1941 reorder_face_derivative_indices<dim>(
1942 faces[face].exterior_face_no,
1943 e,
1944 cell_it->reference_cell());
1945 face_data.general_jac[n_q_points + q][d][e][v] =
1946 inv_jac[d][ee];
1947 }
1948 for (unsigned int d = 0; d < dim; ++d)
1949 for (unsigned int e = 0; e < dim; ++e)
1950 if (e != d &&
1951 std::abs(inv_jac[d][e]) >
1952 2048 *
1953 std::numeric_limits<double>::epsilon() *
1954 compare_norm_jac)
1955 cell_is_cartesian = false;
1956 }
1957 }
1958 // Fill up with 'known' values
1959 else if (is_boundary_face == false)
1960 {
1961 Assert(faces[face].cells_exterior[0] !=
1962 numbers::invalid_unsigned_int,
1963 ExcInternalError());
1964 for (unsigned int q = 0; q < n_q_points; ++q)
1965 for (unsigned int d = 0; d < dim; ++d)
1966 for (unsigned int e = 0; e < dim; ++e)
1967 face_data.general_jac[n_q_points + q][d][e][v] =
1968 face_data.general_jac[n_q_points + q][d][e][0];
1969 }
1970 // If boundary face, simply set the data to zero (will not
1971 // be used). Note that faces over periodic boundary
1972 // conditions will be treated as interior ones in this setup.
1973 else
1974 for (unsigned int q = 0; q < n_q_points; ++q)
1975 for (unsigned int d = 0; d < dim; ++d)
1976 for (unsigned int e = 0; e < dim; ++e)
1977 face_data.general_jac[n_q_points + q][d][e][v] = 0.;
1978 }
1979
1980 // check if face is affine or at least if it is flat
1981 // (i.e., all normal vectors are the same)
1982 if (my_q == 0)
1983 {
1984 GeometryType face_type = affine;
1985 if (JxW_is_similar == false)
1986 face_type = flat_faces;
1987 if (normal_is_similar == false)
1988 face_type = general;
1989 if (face_type == affine && cell_is_cartesian)
1990 face_type = cartesian;
1991 mapping_info.face_type[face] = face_type;
1992 }
1993
1994 // Fill in quadrature points
1995 if (fe_face_values.get_update_flags() & update_quadrature_points)
1996 {
1997 data.first[my_q].quadrature_point_offsets.push_back(
1998 data.first[my_q].quadrature_points.size());
1999 if (fe_face_values.get_update_flags() &
2000 update_quadrature_points)
2001 for (unsigned int q = 0; q < n_q_points; ++q)
2002 data.first[my_q].quadrature_points.push_back(
2003 face_data.quadrature_points[q]);
2004 }
2005
2006 using VEC_ARRAY = Tensor<1, VectorizedArrayType::size(), Number>;
2007 unsigned int insert_position = data.first[my_q].JxW_values.size();
2008
2009 // Fill in JxW values, apply compression
2010 if (mapping_info.face_type[face] <= affine)
2011 {
2012 if (my_q == 0)
2013 {
2014 // find out if we already had the same JxW values before
2015 std::pair<Tensor<1, 2 * dim * dim + dim + 1, VEC_ARRAY>,
2016 unsigned int>
2017 new_entry;
2018 new_entry.second = data.second.data.size();
2019 for (unsigned int v = 0; v < VectorizedArrayType::size();
2020 ++v)
2021 new_entry.first[2 * dim * dim + dim][v] =
2022 face_data.JxW_values[0][v] /
2023 fe_face_values.get_quadrature().weight(0) /
2024 Utilities::fixed_power<dim>(face_data.jac_size);
2025
2026 new_entry.second = data.second.data.size();
2027 for (unsigned int d = 0; d < dim; ++d)
2028 for (unsigned int e = 0; e < dim; ++e)
2029 for (unsigned int v = 0;
2030 v < VectorizedArrayType::size();
2031 ++v)
2032 new_entry.first[d * dim + e][v] =
2033 face_data.general_jac[0][d][e][v];
2034 if (is_boundary_face == false)
2035 for (unsigned int d = 0; d < dim; ++d)
2036 for (unsigned int e = 0; e < dim; ++e)
2037 for (unsigned int v = 0;
2038 v < VectorizedArrayType::size();
2039 ++v)
2040 new_entry.first[dim * dim + d * dim + e][v] =
2041 face_data.general_jac[n_q_points][d][e][v];
2042 // we need to add the normal vector here because we
2043 // store both the inverse jacobian and the normal vector
2044 // times the Jacobian; of course, there will be
2045 // different values in their product for normal vectors
2046 // oriented in different ways (the memory saving is
2047 // still significant); we need to divide by the Jacobian
2048 // size to get the right scaling
2049 for (unsigned int d = 0; d < dim; ++d)
2050 for (unsigned int v = 0;
2051 v < VectorizedArrayType::size();
2052 ++v)
2053 new_entry.first[2 * dim * dim + d][v] =
2054 face_data.normal_vectors[0][d][v] /
2055 face_data.jac_size;
2056
2057 insert_position =
2058 data.second.data.insert(new_entry).first->second;
2059 }
2060 else
2061 insert_position = data.first[0].data_index_offsets[face];
2062 }
2063 data.first[my_q].data_index_offsets.push_back(insert_position);
2064 if (mapping_info.face_type[face] > affine)
2065 {
2066 for (unsigned int q = 0; q < n_q_points; ++q)
2067 data.first[my_q].JxW_values.push_back(
2068 face_data.JxW_values[q]);
2069 for (unsigned int q = 0; q < n_q_points; ++q)
2070 data.first[my_q].normal_vectors.push_back(
2071 face_data.normal_vectors[q]);
2072 for (unsigned int q = 0; q < n_q_points; ++q)
2073 data.first[my_q].jacobians[0].push_back(
2074 face_data.general_jac[q]);
2075 if (is_boundary_face == false)
2076 for (unsigned int q = 0; q < n_q_points; ++q)
2077 data.first[my_q].jacobians[1].push_back(
2078 face_data.general_jac[n_q_points + q]);
2079 }
2080 }
2081 }
2082
2083 template <int dim, typename Number, typename VectorizedArrayType>
2084 void
2085 compute_normal_times_jacobian(
2086 const unsigned int first_face,
2087 const unsigned int last_face,
2088 const std::vector<GeometryType> &face_type,
2089 const std::vector<FaceToCellTopology<VectorizedArrayType::size()>>
2090 &faces,
2091 MappingInfoStorage<dim - 1, dim, VectorizedArrayType> &data_faces)
2092 {
2093 for (unsigned int face = first_face; face < last_face; ++face)
2094 {
2095 const bool is_boundary_face =
2096 faces[face].cells_exterior[0] == numbers::invalid_unsigned_int;
2097 const unsigned int n_q_points_work =
2098 face_type[face] > affine ? data_faces.descriptor[0].n_q_points :
2099 1;
2100 const unsigned int offset = data_faces.data_index_offsets[face];
2101
2102 for (unsigned int q = 0; q < n_q_points_work; ++q)
2103 {
2104 data_faces.normals_times_jacobians[0][offset + q] =
2105 data_faces.normal_vectors[offset + q] *
2106 data_faces.jacobians[0][offset + q];
2107 if (is_boundary_face == false)
2108 data_faces.normals_times_jacobians[1][offset + q] =
2109 data_faces.normal_vectors[offset + q] *
2110 data_faces.jacobians[1][offset + q];
2111 }
2112 }
2113 }
2114
2115
2116
2117 /**
2118 * This function computes and tabulates the mapping information for
2119 * MappingQ-derived mappings on a range of faces calling into the tensor
2120 * product face evaluators interpolators of the matrix-free framework,
2121 * using a polynomial expansion of the cell geometry underlying the
2122 * MappingQ class.
2123 */
2124 template <int dim,
2125 typename Number,
2126 typename VectorizedArrayType,
2127 typename VectorizedDouble>
2128 void
2129 mapping_q_compute_range(
2130 const unsigned int begin_face,
2131 const unsigned int end_face,
2132 const std::vector<FaceToCellTopology<VectorizedArrayType::size()>>
2133 &faces,
2134 const std::vector<GeometryType> &face_type,
2135 const std::vector<bool> &process_face,
2136 const UpdateFlags update_flags_faces,
2137 const AlignedVector<double> &plain_quadrature_points,
2138 const ShapeInfo<double> &shape_info,
2139 MappingInfoStorage<dim - 1, dim, VectorizedArrayType> &my_data)
2140 {
2141 constexpr unsigned int n_lanes = VectorizedArrayType::size();
2142 constexpr unsigned int n_lanes_d = VectorizedDouble::size();
2143
2144 const unsigned int n_q_points = my_data.descriptor[0].n_q_points;
2145 const unsigned int n_mapping_points =
2146 shape_info.dofs_per_component_on_cell;
2147 constexpr unsigned int hess_dim = dim * (dim + 1) / 2;
2148
2149 FEEvaluationData<dim, VectorizedDouble, true> eval_int(shape_info,
2150 true);
2151 FEEvaluationData<dim, VectorizedDouble, true> eval_ext(shape_info,
2152 false);
2153
2154 AlignedVector<VectorizedDouble> evaluation_data_int,
2155 evaluation_data_ext;
2156 eval_int.set_data_pointers(&evaluation_data_int, dim);
2157 eval_ext.set_data_pointers(&evaluation_data_ext, dim);
2158
2159 for (unsigned int face = begin_face; face < end_face; ++face)
2160 for (unsigned vv = 0; vv < n_lanes; vv += n_lanes_d)
2161 {
2162 FaceToCellTopology<VectorizedDouble::size()> face_double = {};
2163 face_double.interior_face_no = faces[face].interior_face_no;
2164 face_double.exterior_face_no = faces[face].exterior_face_no;
2165 face_double.face_orientation = faces[face].face_orientation;
2166 face_double.subface_index = faces[face].subface_index;
2167
2168 // load the geometry field for all SIMD lanes
2169 unsigned int start_indices[n_lanes_d];
2170 for (unsigned int v = 0; v < n_lanes_d; ++v)
2171 if (faces[face].cells_interior[vv + v] !=
2172 numbers::invalid_unsigned_int)
2173 start_indices[v] =
2174 faces[face].cells_interior[vv + v] * n_mapping_points * dim;
2175 else
2176 start_indices[v] =
2177 faces[face].cells_interior[0] * n_mapping_points * dim;
2178
2179 eval_int.reinit_face(face_double);
2180 vectorized_load_and_transpose(n_mapping_points * dim,
2181 plain_quadrature_points.data(),
2182 start_indices,
2183 eval_int.begin_dof_values());
2184
2185 // now let the matrix-free evaluators provide us with the
2186 // data on faces
2187 FEFaceEvaluationFactory<dim, VectorizedDouble>::evaluate(
2188 dim,
2189 EvaluationFlags::values | EvaluationFlags::gradients |
2190 (update_flags_faces & update_jacobian_grads ?
2191 EvaluationFlags::hessians :
2192 EvaluationFlags::nothing),
2193 eval_int.begin_dof_values(),
2194 eval_int);
2195
2196 if (update_flags_faces & update_quadrature_points)
2197 for (unsigned int q = 0; q < n_q_points; ++q)
2198 for (unsigned int d = 0; d < dim; ++d)
2199 store_vectorized_array(
2200 eval_int.begin_values()[d * n_q_points + q],
2201 vv,
2202 my_data.quadrature_points
2203 [my_data.quadrature_point_offsets[face] + q][d]);
2204
2205 if (process_face[face] == false)
2206 continue;
2207
2208 const unsigned int offset = my_data.data_index_offsets[face];
2209
2210 const auto compute_jacobian_grad =
2211 [&](unsigned int face_no,
2212 int is_exterior,
2213 unsigned int q,
2214 Tensor<2, dim, VectorizedDouble> inv_jac,
2215 FEEvaluationData<dim, VectorizedDouble, true> &eval) {
2216 Tensor<2, dim, VectorizedDouble> inv_transp_jac_permutation;
2217 for (unsigned int d = 0; d < dim; ++d)
2218 for (unsigned int e = 0; e < dim; ++e)
2219 {
2220 const unsigned int ee =
2221 ExtractFaceHelper::reorder_face_derivative_indices<
2222 dim>(face_no, e);
2223 inv_transp_jac_permutation[d][e] = inv_jac[ee][d];
2224 }
2225 Tensor<2, dim, VectorizedDouble> jacobi;
2226 for (unsigned int e = 0; e < dim; ++e)
2227 for (unsigned int d = 0; d < dim; ++d)
2228 jacobi[d][e] =
2229 eval.begin_gradients()[(d * n_q_points + q) * dim + e];
2230 Tensor<2, dim, VectorizedDouble> inv_transp_jac =
2231 transpose(invert(jacobi));
2232 Tensor<3, dim, VectorizedDouble> jac_grad;
2233 for (unsigned int d = 0; d < dim; ++d)
2234 {
2235 for (unsigned int e = 0; e < dim; ++e)
2236 jac_grad[d][e][e] =
2237 eval.begin_hessians()[q + (d * hess_dim + e) *
2238 n_q_points];
2239 for (unsigned int c = dim, e = 0; e < dim; ++e)
2240 for (unsigned int f = e + 1; f < dim; ++f, ++c)
2241 jac_grad[d][e][f] = jac_grad[d][f][e] =
2242 eval.begin_hessians()[q + (d * hess_dim + c) *
2243 n_q_points];
2244 const auto inv_jac_grad =
2245 process_jacobian_gradient(inv_transp_jac_permutation,
2246 inv_transp_jac,
2247 jac_grad);
2248 for (unsigned int e = 0; e < dim; ++e)
2249 {
2250 for (unsigned int d = 0; d < hess_dim; ++d)
2251 store_vectorized_array(
2252 inv_jac_grad[d][e],
2253 vv,
2254 my_data.jacobian_gradients[is_exterior]
2255 [offset + q][d][e]);
2256 }
2257
2258 // Also store the non-inverse Jacobian gradient.
2259 // the diagonal part of Jacobian gradient comes first.
2260 // jac_grad already has its derivatives reordered,
2261 // so no need to compensate for this here
2262 for (unsigned int d = 0; d < dim; ++d)
2263 for (unsigned int f = 0; f < dim; ++f)
2264 store_vectorized_array(
2265 jac_grad[f][d][d],
2266 vv,
2267 my_data.jacobian_gradients_non_inverse[is_exterior]
2268 [offset + q]
2269 [d][f]);
2270
2271 // then the upper-diagonal part
2272 for (unsigned int d = 0, count = dim; d < dim; ++d)
2273 for (unsigned int e = d + 1; e < dim; ++e, ++count)
2274 for (unsigned int f = 0; f < dim; ++f)
2275 store_vectorized_array(
2276 jac_grad[f][d][e],
2277 vv,
2278 my_data.jacobian_gradients_non_inverse
2279 [is_exterior][offset + q][count][f]);
2280 }
2281 };
2282
2283 // go through the faces and fill the result
2284 const unsigned int n_points_compute =
2285 face_type[face] <= affine ? 1 : n_q_points;
2286 for (unsigned int q = 0; q < n_points_compute; ++q)
2287 {
2288 const unsigned int interior_face_no =
2289 faces[face].interior_face_no;
2290 Tensor<2, dim, VectorizedDouble> jac;
2291 for (unsigned int e = 0; e < dim; ++e)
2292 {
2293 const unsigned int ee =
2294 ExtractFaceHelper::reorder_face_derivative_indices<dim>(
2295 interior_face_no, e);
2296 for (unsigned int d = 0; d < dim; ++d)
2297 jac[d][ee] =
2298 eval_int
2299 .begin_gradients()[(d * n_q_points + q) * dim + e];
2300 }
2301 Tensor<2, dim, VectorizedDouble> inv_jac = invert(jac);
2302 for (unsigned int e = 0; e < dim; ++e)
2303 {
2304 const unsigned int ee =
2305 ExtractFaceHelper::reorder_face_derivative_indices<dim>(
2306 interior_face_no, e);
2307 for (unsigned int d = 0; d < dim; ++d)
2308 store_vectorized_array(
2309 inv_jac[ee][d],
2310 vv,
2311 my_data.jacobians[0][offset + q][d][e]);
2312 }
2313 if (face_type[face] <= affine)
2314 for (unsigned int e = 0; e < dim; ++e)
2315 {
2316 const unsigned int ee =
2317 ExtractFaceHelper::reorder_face_derivative_indices<
2318 dim>(interior_face_no, e);
2319 for (unsigned int d = 0; d < dim; ++d)
2320 store_vectorized_array(
2321 jac[d][ee],
2322 vv,
2323 my_data.jacobians[0][offset + q + 1][d][e]);
2324 }
2325
2326 if (update_flags_faces & update_jacobian_grads)
2327 {
2328 compute_jacobian_grad(
2329 interior_face_no, 0, q, inv_jac, eval_int);
2330 }
2331
2332 std::array<Tensor<1, dim, VectorizedDouble>, dim - 1>
2333 tangential_vectors;
2334 for (unsigned int d = 0; d != dim - 1; ++d)
2335 for (unsigned int e = 0; e < dim; ++e)
2336 for (unsigned int f = 0; f < dim; ++f)
2337 tangential_vectors[d][e] +=
2338 jac[e][f] * GeometryInfo<dim>::unit_tangential_vectors
2339 [interior_face_no][d][f];
2340
2341 Tensor<1, dim, VectorizedDouble> boundary_form;
2342 if (dim == 1)
2343 boundary_form[0] = interior_face_no == 0 ? -1. : 1.;
2344 else if (dim == 2)
2345 boundary_form = cross_product_2d(tangential_vectors[0]);
2346 else if (dim == 3)
2347 boundary_form = cross_product_3d(tangential_vectors[0],
2348 tangential_vectors[1]);
2349 else
2350 DEAL_II_NOT_IMPLEMENTED();
2351
2352 const VectorizedDouble JxW =
2353 boundary_form.norm() *
2354 (face_type[face] <= affine ?
2355 1. :
2356 my_data.descriptor[0].quadrature.weight(q));
2357
2358 if constexpr (running_in_debug_mode())
2359 {
2360 for (unsigned int v = 0; v < n_lanes_d; ++v)
2361 Assert(JxW[v] > 0.0, ExcInternalError());
2362 }
2363
2364 store_vectorized_array(JxW,
2365 vv,
2366 my_data.JxW_values[offset + q]);
2367
2368 const Tensor<1, dim, VectorizedDouble> normal =
2369 boundary_form / boundary_form.norm();
2370
2371 for (unsigned int d = 0; d < dim; ++d)
2372 store_vectorized_array(
2373 normal[d], vv, my_data.normal_vectors[offset + q][d]);
2374
2375 my_data.normals_times_jacobians[0][offset + q] =
2376 my_data.normal_vectors[offset + q] *
2377 my_data.jacobians[0][offset + q];
2378 }
2379
2380 if (faces[face].cells_exterior[0] !=
2381 numbers::invalid_unsigned_int)
2382 {
2383 for (unsigned int v = 0; v < n_lanes_d; ++v)
2384 if (faces[face].cells_exterior[vv + v] !=
2385 numbers::invalid_unsigned_int)
2386 start_indices[v] = faces[face].cells_exterior[vv + v] *
2387 n_mapping_points * dim;
2388 else
2389 start_indices[v] =
2390 faces[face].cells_exterior[0] * n_mapping_points * dim;
2391
2392 eval_ext.reinit_face(face_double);
2393 vectorized_load_and_transpose(n_mapping_points * dim,
2394 plain_quadrature_points.data(),
2395 start_indices,
2396 eval_ext.begin_dof_values());
2397
2398 FEFaceEvaluationFactory<dim, VectorizedDouble>::evaluate(
2399 dim,
2400 EvaluationFlags::values | EvaluationFlags::gradients |
2401 (update_flags_faces & update_jacobian_grads ?
2402 EvaluationFlags::hessians :
2403 EvaluationFlags::nothing),
2404 eval_ext.begin_dof_values(),
2405 eval_ext);
2406
2407 for (unsigned int q = 0; q < n_points_compute; ++q)
2408 {
2409 const unsigned int exterior_face_no =
2410 faces[face].exterior_face_no;
2411 Tensor<2, dim, VectorizedDouble> jac;
2412 for (unsigned int e = 0; e < dim; ++e)
2413 {
2414 const unsigned int ee =
2415 ExtractFaceHelper::reorder_face_derivative_indices<
2416 dim>(exterior_face_no, e);
2417 for (unsigned int d = 0; d < dim; ++d)
2418 jac[d][ee] =
2419 eval_ext
2420 .begin_gradients()[(d * n_q_points + q) * dim +
2421 e];
2422 }
2423 Tensor<2, dim, VectorizedDouble> inv_jac = invert(jac);
2424 for (unsigned int e = 0; e < dim; ++e)
2425 {
2426 const unsigned int ee =
2427 ExtractFaceHelper::reorder_face_derivative_indices<
2428 dim>(exterior_face_no, e);
2429 for (unsigned int d = 0; d < dim; ++d)
2430 store_vectorized_array(
2431 inv_jac[ee][d],
2432 vv,
2433 my_data.jacobians[1][offset + q][d][e]);
2434 }
2435 if (face_type[face] <= affine)
2436 for (unsigned int e = 0; e < dim; ++e)
2437 {
2438 const unsigned int ee = ExtractFaceHelper::
2439 reorder_face_derivative_indices<dim>(
2440 exterior_face_no, e);
2441 for (unsigned int d = 0; d < dim; ++d)
2442 store_vectorized_array(
2443 jac[d][ee],
2444 vv,
2445 my_data.jacobians[1][offset + q + 1][d][e]);
2446 }
2447
2448 if (update_flags_faces & update_jacobian_grads)
2449 {
2450 compute_jacobian_grad(
2451 exterior_face_no, 1, q, inv_jac, eval_ext);
2452 }
2453
2454 my_data.normals_times_jacobians[1][offset + q] =
2455 my_data.normal_vectors[offset + q] *
2456 my_data.jacobians[1][offset + q];
2457 }
2458 }
2459 }
2460 }
2461
2462 } // namespace ExtractFaceHelper
2463
2464
2465
2466 template <int dim, typename Number, typename VectorizedArrayType>
2467 void
2468 MappingInfo<dim, Number, VectorizedArrayType>::initialize_faces(
2469 const dealii::Triangulation<dim> &tria,
2470 const std::vector<std::pair<unsigned int, unsigned int>> &cells,
2471 const std::vector<FaceToCellTopology<VectorizedArrayType::size()>> &faces,
2472 const std::vector<unsigned int> &active_fe_index,
2473 const dealii::hp::MappingCollection<dim> &mapping)
2474 {
2475 face_type.resize(faces.size(), general);
2476
2477 if (faces.empty())
2478 return;
2479
2480 // Create as many chunks of cells as we have threads and spawn the
2481 // work
2482 unsigned int work_per_chunk =
2483 std::max(std::size_t(8),
2484 (faces.size() + MultithreadInfo::n_threads() - 1) /
2485 MultithreadInfo::n_threads());
2486
2487 std::vector<std::pair<
2488 std::vector<MappingInfoStorage<dim - 1, dim, VectorizedArrayType>>,
2489 ExtractFaceHelper::
2490 CompressedFaceData<dim, Number, VectorizedArrayType>>>
2491 data_faces_local;
2492 // Reserve enough space to avoid re-allocation (which would destroy
2493 // the references passed to the tasks!)
2494 data_faces_local.reserve(MultithreadInfo::n_threads());
2495
2496 {
2497 Threads::TaskGroup<> tasks;
2498 std::pair<unsigned int, unsigned int> face_range(0U, work_per_chunk);
2499 while (face_range.first < faces.size())
2500 {
2501 data_faces_local.push_back(std::make_pair(
2502 std::vector<
2503 MappingInfoStorage<dim - 1, dim, VectorizedArrayType>>(
2504 face_data.size()),
2505 ExtractFaceHelper::
2506 CompressedFaceData<dim, Number, VectorizedArrayType>(
2507 ExtractCellHelper::get_jacobian_size(tria))));
2508 tasks += Threads::new_task(
2509 &ExtractFaceHelper::
2510 initialize_face_range<dim, Number, VectorizedArrayType>,
2511 face_range,
2512 tria,
2513 cells,
2514 faces,
2515 active_fe_index,
2516 mapping,
2517 *this,
2518 data_faces_local.back());
2519 face_range.first = face_range.second;
2520 face_range.second += work_per_chunk;
2521 }
2522
2523 tasks.join_all();
2524 }
2525
2526 // Fill in each thread's constant data (normals, JxW, normals x
2527 // Jacobian) into the data of the zeroth chunk in serial
2528 std::vector<std::vector<unsigned int>> indices_compressed(
2529 data_faces_local.size());
2530 for (unsigned int i = 0; i < data_faces_local.size(); ++i)
2531 ExtractCellHelper::merge_compressed_data(
2532 data_faces_local[i].second.data,
2533 data_faces_local[0].second.data,
2534 indices_compressed[i]);
2535
2536 const UpdateFlags update_flags_common =
2537 update_flags_boundary_faces | update_flags_inner_faces;
2538
2539 // Collect all data in the final data fields.
2540 // First allocate the memory
2541 const unsigned int n_constant_data =
2542 data_faces_local[0].second.data.size();
2543 for (unsigned int my_q = 0; my_q < face_data.size(); ++my_q)
2544 {
2545 face_data[my_q].data_index_offsets.resize(face_type.size());
2546 std::vector<std::array<std::size_t, 2>> shift(
2547 data_faces_local.size());
2548 shift[0][0] = n_constant_data;
2549 shift[0][1] = 0;
2550 for (unsigned int i = 1; i < data_faces_local.size(); ++i)
2551 {
2552 shift[i][0] =
2553 shift[i - 1][0] +
2554 data_faces_local[i - 1].first[my_q].JxW_values.size();
2555 shift[i][1] =
2556 shift[i - 1][1] +
2557 data_faces_local[i - 1].first[my_q].quadrature_points.size();
2558 }
2559 face_data[my_q].JxW_values.resize_fast(
2560 shift.back()[0] +
2561 data_faces_local.back().first[my_q].JxW_values.size());
2562 face_data[my_q].normal_vectors.resize_fast(
2563 face_data[my_q].JxW_values.size());
2564 face_data[my_q].jacobians[0].resize_fast(
2565 face_data[my_q].JxW_values.size());
2566 face_data[my_q].jacobians[1].resize_fast(
2567 face_data[my_q].JxW_values.size());
2568 if (update_flags_common & update_jacobian_grads)
2569 {
2570 face_data[my_q].jacobian_gradients[0].resize_fast(
2571 face_data[my_q].JxW_values.size());
2572 face_data[my_q].jacobian_gradients[1].resize_fast(
2573 face_data[my_q].JxW_values.size());
2574 face_data[my_q].jacobian_gradients_non_inverse[0].resize_fast(
2575 face_data[my_q].JxW_values.size());
2576 face_data[my_q].jacobian_gradients_non_inverse[1].resize_fast(
2577 face_data[my_q].JxW_values.size());
2578 }
2579 face_data[my_q].normals_times_jacobians[0].resize_fast(
2580 face_data[my_q].JxW_values.size());
2581 face_data[my_q].normals_times_jacobians[1].resize_fast(
2582 face_data[my_q].JxW_values.size());
2583 if (update_flags_common & update_quadrature_points)
2584 {
2585 face_data[my_q].quadrature_point_offsets.resize(face_type.size());
2586 face_data[my_q].quadrature_points.resize_fast(
2587 shift.back()[1] +
2588 data_faces_local.back().first[my_q].quadrature_points.size());
2589 }
2590
2591 // start the tasks to gather the data in parallel
2592 Threads::TaskGroup<> tasks;
2593 for (unsigned int i = 0; i < data_faces_local.size(); ++i)
2594 tasks += Threads::new_task(
2595 &ExtractCellHelper::
2596 copy_data<dim - 1, dim, Number, VectorizedArrayType>,
2597 work_per_chunk * i,
2598 shift[i],
2599 indices_compressed[i],
2600 face_type,
2601 data_faces_local[i].first[my_q],
2602 face_data[my_q]);
2603
2604 // fill the constant data fields (in parallel to the loop above)
2605 if (my_q == 0)
2606 {
2607 const Number jac_size =
2608 ExtractCellHelper::get_jacobian_size(tria);
2609 for (const auto &it : data_faces_local[0].second.data)
2610 {
2611 // JxW values; invert previously applied scaling
2612 for (unsigned int v = 0; v < VectorizedArrayType::size(); ++v)
2613 face_data[my_q].JxW_values[it.second][v] =
2614 it.first[2 * dim * dim + dim][v] *
2615 Utilities::fixed_power<dim>(jac_size);
2616
2617 // inverse Jacobians
2618 for (unsigned int i = 0; i < 2; ++i)
2619 for (unsigned int d = 0; d < dim; ++d)
2620 for (unsigned int e = 0; e < dim; ++e)
2621 for (unsigned int v = 0;
2622 v < VectorizedArrayType::size();
2623 ++v)
2624 face_data[my_q].jacobians[i][it.second][d][e][v] =
2625 it.first[i * dim * dim + d * dim + e][v];
2626
2627 // normal vectors; invert previously applied scaling
2628 for (unsigned int d = 0; d < dim; ++d)
2629 for (unsigned int v = 0; v < VectorizedArrayType::size();
2630 ++v)
2631 face_data[my_q].normal_vectors[it.second][d][v] =
2632 it.first[2 * dim * dim + d][v] * jac_size;
2633 }
2634 }
2635 else
2636 {
2637 for (unsigned int i = 0; i < n_constant_data; ++i)
2638 {
2639 face_data[my_q].JxW_values[i] = face_data[0].JxW_values[i];
2640 face_data[my_q].normal_vectors[i] =
2641 face_data[0].normal_vectors[i];
2642 for (unsigned k = 0; k < 2; ++k)
2643 face_data[my_q].jacobians[k][i] =
2644 face_data[0].jacobians[k][i];
2645 }
2646 }
2647
2648 // ... wait for the parallel work to finish
2649 tasks.join_all();
2650
2651 // finally compute the normal times the jacobian
2652 for (unsigned int i = 0; i < data_faces_local.size(); ++i)
2653 tasks += Threads::new_task(
2654 &ExtractFaceHelper::
2655 compute_normal_times_jacobian<dim, Number, VectorizedArrayType>,
2656 work_per_chunk * i,
2657 std::min<unsigned int>(work_per_chunk * (i + 1), faces.size()),
2658 face_type,
2659 faces,
2660 face_data[my_q]);
2661 tasks.join_all();
2662 }
2663 }
2664
2665
2666
2667 template <int dim, typename Number, typename VectorizedArrayType>
2668 void
2669 MappingInfo<dim, Number, VectorizedArrayType>::compute_mapping_q(
2670 const dealii::Triangulation<dim> &tria,
2671 const std::vector<std::pair<unsigned int, unsigned int>> &cell_array,
2672 const FaceInfo<VectorizedArrayType::size()> &face_info)
2673 {
2674 // step 1: extract quadrature point data with the data appropriate for
2675 // MappingQ
2676 AssertDimension(this->mapping_collection->size(), 1);
2677
2678 const MappingQ<dim> *mapping_q = dynamic_cast<const MappingQ<dim> *>(
2679 &this->mapping_collection->operator[](0));
2680 Assert(mapping_q != nullptr, ExcInternalError());
2681
2682 const unsigned int mapping_degree = mapping_q->get_degree();
2683 const unsigned int n_mapping_points =
2684 Utilities::pow(mapping_degree + 1, dim);
2685 AlignedVector<double> plain_quadrature_points(cell_array.size() *
2686 n_mapping_points * dim);
2687
2688 const double jacobian_size = ExtractCellHelper::get_jacobian_size(tria);
2689
2690 std::vector<unsigned int> cell_data_index;
2691 std::vector<GeometryType> preliminary_cell_type(cell_array.size());
2692 {
2693 AlignedVector<std::array<Tensor<2, dim>, dim + 1>> jacobians_on_stencil(
2694 cell_array.size());
2695
2696 // Create as many chunks of cells as we have threads and spawn the
2697 // work
2698 unsigned int work_per_chunk =
2699 std::max(std::size_t(1),
2700 (cell_array.size() + MultithreadInfo::n_threads() - 1) /
2701 MultithreadInfo::n_threads());
2702
2703 // we manually use tasks here rather than parallel::apply_to_subranges
2704 // because we want exactly as many loops as we have threads - the
2705 // initialization of the loops with FEValues is expensive
2706 std::size_t offset = 0;
2707 Threads::TaskGroup<> tasks;
2708 for (unsigned int t = 0; t < MultithreadInfo::n_threads();
2709 ++t, offset += work_per_chunk)
2710 tasks += Threads::new_task(
2711 &ExtractCellHelper::mapping_q_query_fe_values<dim>,
2712 offset,
2713 std::min(cell_array.size(), offset + work_per_chunk),
2714 *mapping_q,
2715 tria,
2716 cell_array,
2717 jacobian_size,
2718 preliminary_cell_type,
2719 plain_quadrature_points,
2720 jacobians_on_stencil);
2721 tasks.join_all();
2722 cell_data_index =
2723 ExtractCellHelper::mapping_q_find_compression(jacobian_size,
2724 jacobians_on_stencil,
2725 n_mapping_points,
2726 plain_quadrature_points,
2727 preliminary_cell_type);
2728 }
2729
2730 // step 2: compute the appropriate evaluation matrices for cells and
2731 // faces
2732
2733 // We want to use vectorization for computing the quantities, but must
2734 // evaluate the geometry in double precision; thus, for floats we need
2735 // to do things in two sweeps and convert the final result.
2736 constexpr unsigned int n_lanes = VectorizedArrayType::size();
2737
2738 using VectorizedDouble =
2739 VectorizedArray<double,
2740 ((std::is_same_v<Number, float> &&
2741 VectorizedArrayType::size() > 1) ?
2742 VectorizedArrayType::size() / 2 :
2743 VectorizedArrayType::size())>;
2744
2745 // Create a ShapeInfo object to provide the necessary interpolators to
2746 // the various quadrature points. Note that it is initialized with the
2747 // finite element fe_geometry using the degree of the mapping, which is
2748 // not the same as the degree of the underlying finite element shape
2749 // functions or the quadrature points; shape info is merely a vehicle to
2750 // return us the right interpolation matrices from the cell support
2751 // points to the cell and face quadrature points.
2752 std::vector<ShapeInfo<double>> shape_infos(cell_data.size());
2753 {
2754 FE_DGQ<dim> fe_geometry(mapping_degree);
2755 for (unsigned int my_q = 0; my_q < cell_data.size(); ++my_q)
2756 shape_infos[my_q].reinit(cell_data[my_q].descriptor[0].quadrature,
2757 fe_geometry);
2758 }
2759
2760 // step 3: find compression of cells with vectorization
2761 std::map<std::array<unsigned int, n_lanes>, unsigned int> compressed_data;
2762
2763 cell_type.resize(cell_array.size() / n_lanes);
2764 std::vector<bool> process_cell(cell_type.size());
2765 std::vector<unsigned int> cell_data_index_vect(cell_type.size());
2766
2767 for (unsigned int cell = 0; cell < cell_array.size(); cell += n_lanes)
2768 {
2769 std::pair<std::array<unsigned int, n_lanes>, unsigned int>
2770 data_indices;
2771 for (unsigned int i = 0; i < n_lanes; ++i)
2772 data_indices.first[i] = cell_data_index[cell + i];
2773 data_indices.second = cell / n_lanes;
2774
2775 auto inserted = compressed_data.insert(data_indices);
2776
2777 process_cell[cell / n_lanes] = inserted.second;
2778 if (inserted.second == true)
2779 cell_data_index_vect[cell / n_lanes] = data_indices.second;
2780 else
2781 cell_data_index_vect[cell / n_lanes] = inserted.first->second;
2782
2783 cell_type[cell / n_lanes] =
2784 *std::max_element(preliminary_cell_type.data() + cell,
2785 preliminary_cell_type.data() + cell + n_lanes);
2786 }
2787
2788 // step 4: compute the data on cells from the cached quadrature
2789 // points, filling up all SIMD lanes as appropriate
2790 for (unsigned int my_q = 0; my_q < cell_data.size(); ++my_q)
2791 {
2792 MappingInfoStorage<dim, dim, VectorizedArrayType> &my_data =
2793 cell_data[my_q];
2794
2795 // step 4a: set the index offsets, find out how much to allocate,
2796 // and allocate the memory
2797 const unsigned int n_q_points = my_data.descriptor[0].n_q_points;
2798 unsigned int max_size = 0;
2799 my_data.data_index_offsets.resize(cell_type.size());
2800 for (unsigned int cell = 0; cell < cell_type.size(); ++cell)
2801 {
2802 if (process_cell[cell] == false)
2803 my_data.data_index_offsets[cell] =
2804 my_data.data_index_offsets[cell_data_index_vect[cell]];
2805 else
2806 my_data.data_index_offsets[cell] = max_size;
2807 max_size =
2808 std::max(max_size,
2809 my_data.data_index_offsets[cell] +
2810 (cell_type[cell] <= affine ? 2 : n_q_points));
2811 }
2812
2813 my_data.JxW_values.resize_fast(max_size);
2814 my_data.jacobians[0].resize_fast(max_size);
2815 if (update_flags_cells & update_jacobian_grads)
2816 {
2817 my_data.jacobian_gradients[0].resize_fast(max_size);
2818 my_data.jacobian_gradients_non_inverse[0].resize_fast(max_size);
2819 }
2820
2821 if (update_flags_cells & update_quadrature_points)
2822 {
2823 my_data.quadrature_point_offsets.resize(cell_type.size());
2824 for (unsigned int cell = 1; cell < cell_type.size(); ++cell)
2825 if (cell_type[cell - 1] <= affine)
2826 my_data.quadrature_point_offsets[cell] =
2827 my_data.quadrature_point_offsets[cell - 1] + 1;
2828 else
2829 my_data.quadrature_point_offsets[cell] =
2830 my_data.quadrature_point_offsets[cell - 1] + n_q_points;
2831 my_data.quadrature_points.resize_fast(
2832 my_data.quadrature_point_offsets.back() +
2833 (cell_type.back() <= affine ? 1 : n_q_points));
2834 }
2835
2836 // step 4b: go through the cells and compute the information using
2837 // similar evaluators as for the matrix-free integrals
2838 dealii::parallel::apply_to_subranges(
2839 0U,
2840 cell_type.size(),
2841 [&](const unsigned int begin, const unsigned int end) {
2842 ExtractCellHelper::mapping_q_compute_range<dim,
2843 Number,
2844 VectorizedArrayType,
2845 VectorizedDouble>(
2846 begin,
2847 end,
2848 tria,
2849 cell_array,
2850 cell_type,
2851 process_cell,
2852 update_flags_cells,
2853 plain_quadrature_points,
2854 shape_infos[my_q],
2855 my_data);
2856 },
2857 std::max(cell_type.size() / MultithreadInfo::n_threads() / 2,
2858 std::size_t(2U)));
2859 }
2860
2861 const std::vector<FaceToCellTopology<VectorizedArrayType::size()>>
2862 &faces = face_info.faces;
2863 if (faces.empty())
2864 return;
2865
2866 // step 5: find compression of faces with vectorization
2867 std::map<std::array<unsigned int, 2 * n_lanes + 3>, unsigned int>
2868 compressed_faces;
2869
2870 face_type.resize(faces.size());
2871 std::vector<bool> process_face(face_type.size());
2872 std::vector<unsigned int> face_data_index_vect(face_type.size());
2873
2874 for (unsigned int face = 0; face < faces.size(); ++face)
2875 {
2876 std::pair<std::array<unsigned int, 2 * n_lanes + 3>, unsigned int>
2877 data_indices;
2878 for (unsigned int i = 0; i < n_lanes; ++i)
2879 if (faces[face].cells_interior[i] != numbers::invalid_unsigned_int)
2880 data_indices.first[i] =
2881 cell_data_index[faces[face].cells_interior[i]];
2882 else
2883 data_indices.first[i] = data_indices.first[0];
2884 for (unsigned int i = 0; i < n_lanes; ++i)
2885 data_indices.first[n_lanes + i] = data_indices.first[i];
2886 for (unsigned int i = 0; i < n_lanes; ++i)
2887 if (faces[face].cells_exterior[i] != numbers::invalid_unsigned_int)
2888 data_indices.first[n_lanes + i] =
2889 cell_data_index[faces[face].cells_exterior[i]];
2890 data_indices.first[2 * n_lanes] = faces[face].interior_face_no;
2891 data_indices.first[2 * n_lanes + 1] = faces[face].exterior_face_no;
2892 data_indices.first[2 * n_lanes + 2] = faces[face].subface_index;
2893
2894 data_indices.second = face;
2895
2896 auto inserted = compressed_faces.insert(data_indices);
2897
2898 process_face[face] = inserted.second;
2899 if (inserted.second == true)
2900 face_data_index_vect[face] = face;
2901 else
2902 face_data_index_vect[face] = inserted.first->second;
2903
2904 face_type[face] = cartesian;
2905 for (unsigned int i = 0; i < n_lanes; ++i)
2906 if (faces[face].cells_interior[i] != numbers::invalid_unsigned_int)
2907 face_type[face] =
2908 std::max(face_type[face],
2909 preliminary_cell_type[faces[face].cells_interior[i]]);
2910 for (unsigned int i = 0; i < n_lanes; ++i)
2911 if (faces[face].cells_exterior[i] != numbers::invalid_unsigned_int)
2912 face_type[face] =
2913 std::max(face_type[face],
2914 preliminary_cell_type[faces[face].cells_exterior[i]]);
2915 }
2916
2917 // step 6: compute the data on faces from the cached cell quadrature
2918 // points, filling up all SIMD lanes as appropriate
2919 for (unsigned int my_q = 0; my_q < face_data.size(); ++my_q)
2920 {
2921 MappingInfoStorage<dim - 1, dim, VectorizedArrayType> &my_data =
2922 face_data[my_q];
2923
2924 // step 6a: set the index offsets, find out how much to allocate,
2925 // and allocate the memory
2926 const unsigned int n_q_points = my_data.descriptor[0].n_q_points;
2927 unsigned int max_size = 0;
2928 my_data.data_index_offsets.resize(face_type.size());
2929 for (unsigned int face = 0; face < face_type.size(); ++face)
2930 {
2931 if (process_face[face] == false)
2932 my_data.data_index_offsets[face] =
2933 my_data.data_index_offsets[face_data_index_vect[face]];
2934 else
2935 my_data.data_index_offsets[face] = max_size;
2936 max_size =
2937 std::max(max_size,
2938 my_data.data_index_offsets[face] +
2939 (face_type[face] <= affine ? 2 : n_q_points));
2940 }
2941
2942 const UpdateFlags update_flags_common =
2943 update_flags_boundary_faces | update_flags_inner_faces;
2944
2945 my_data.JxW_values.resize_fast(max_size);
2946 my_data.normal_vectors.resize_fast(max_size);
2947 my_data.jacobians[0].resize_fast(max_size);
2948 my_data.jacobians[1].resize_fast(max_size);
2949 if (update_flags_common & update_jacobian_grads)
2950 {
2951 my_data.jacobian_gradients[0].resize_fast(max_size);
2952 my_data.jacobian_gradients[1].resize_fast(max_size);
2953 my_data.jacobian_gradients_non_inverse[0].resize_fast(max_size);
2954 my_data.jacobian_gradients_non_inverse[1].resize_fast(max_size);
2955 }
2956 my_data.normals_times_jacobians[0].resize_fast(max_size);
2957 my_data.normals_times_jacobians[1].resize_fast(max_size);
2958
2959 if (update_flags_common & update_quadrature_points)
2960 {
2961 my_data.quadrature_point_offsets.resize(face_type.size());
2962 my_data.quadrature_point_offsets[0] = 0;
2963 for (unsigned int face = 1; face < faces.size(); ++face)
2964 my_data.quadrature_point_offsets[face] =
2965 n_q_points + my_data.quadrature_point_offsets[face - 1];
2966 my_data.quadrature_points.resize_fast(face_type.size() *
2967 n_q_points);
2968 }
2969
2970 // step 6b: go through the faces and compute the information using
2971 // similar evaluators as for the matrix-free face integrals
2972 dealii::parallel::apply_to_subranges(
2973 0U,
2974 face_type.size(),
2975 [&](const unsigned int begin, const unsigned int end) {
2976 ExtractFaceHelper::mapping_q_compute_range<dim,
2977 Number,
2978 VectorizedArrayType,
2979 VectorizedDouble>(
2980 begin,
2981 end,
2982 faces,
2983 face_type,
2984 process_face,
2985 update_flags_common,
2986 plain_quadrature_points,
2987 shape_infos[my_q],
2988 my_data);
2989 },
2990 std::max(face_type.size() / MultithreadInfo::n_threads() / 2,
2991 std::size_t(2U)));
2992 }
2993
2994 // step 6c: figure out if normal vectors are the same on some of the
2995 // faces which allows us to set the flat_faces face type
2996 unsigned int quad_with_most_points = 0;
2997 for (unsigned int my_q = 1; my_q < face_data.size(); ++my_q)
2998 if (face_data[my_q].descriptor[0].n_q_points >
2999 face_data[quad_with_most_points].descriptor[0].n_q_points)
3000 quad_with_most_points = my_q;
3001 dealii::parallel::apply_to_subranges(
3002 0U,
3003 face_type.size(),
3004 [&](const unsigned int begin, const unsigned int end) {
3005 for (unsigned int face = begin; face < end; ++face)
3006 if (face_type[face] == general)
3007 {
3008 const unsigned int n_q_points =
3009 face_data[quad_with_most_points].descriptor[0].n_q_points;
3010 const Tensor<1, dim, VectorizedArrayType> *normals =
3011 face_data[quad_with_most_points].normal_vectors.data() +
3012 face_data[quad_with_most_points].data_index_offsets[face];
3013 VectorizedArrayType distance = 0.;
3014 for (unsigned int q = 1; q < n_q_points; ++q)
3015 distance += (normals[q] - normals[0]).norm_square();
3016 bool all_small = true;
3017 for (unsigned int v = 0; v < n_lanes; ++v)
3018 if (distance[v] >
3019 50. * std::numeric_limits<Number>::epsilon() *
3020 std::numeric_limits<Number>::epsilon() * n_q_points)
3021 all_small = false;
3022 if (all_small)
3023 face_type[face] = flat_faces;
3024 }
3025 },
3026 std::max(face_type.size() / MultithreadInfo::n_threads() / 2,
3027 std::size_t(2U)));
3028
3029 // step 7: compute the face data by cells. This still needs to be
3030 // transitioned to extracting the information from cell quadrature
3031 // points but we need to figure out the correct indices of neighbors
3032 // within the list of arrays still
3033 initialize_faces_by_cells(tria,
3034 cell_array,
3035 face_info,
3036 *this->mapping_collection);
3037 }
3038
3039
3040
3041 template <int dim, typename Number, typename VectorizedArrayType>
3042 void
3043 MappingInfo<dim, Number, VectorizedArrayType>::initialize_faces_by_cells(
3044 const dealii::Triangulation<dim> &tria,
3045 const std::vector<std::pair<unsigned int, unsigned int>> &cells,
3046 const FaceInfo<VectorizedArrayType::size()> &face_info,
3047 const dealii::hp::MappingCollection<dim> &mapping_in)
3048 {
3049 if (update_flags_faces_by_cells == update_default)
3050 return;
3051
3052
3053 AssertDimension(mapping_in.size(), 1);
3054 const auto &mapping = mapping_in[0];
3055
3056 const unsigned int n_quads = face_data_by_cells.size();
3057 constexpr unsigned int n_lanes = VectorizedArrayType::size();
3058 UpdateFlags update_flags =
3059 (update_flags_faces_by_cells & update_quadrature_points ?
3060 update_quadrature_points :
3061 update_default) |
3062 update_normal_vectors | update_JxW_values | update_jacobians;
3063
3064 const auto compute_neighbor_index =
3065 [&face_info](const unsigned int cell,
3066 const unsigned int face,
3067 const unsigned int lane) {
3068 constexpr unsigned int n_lanes = VectorizedArrayType::size();
3069 const unsigned int cell_this = cell * n_lanes + lane;
3070 unsigned int face_index =
3071 face_info.cell_and_face_to_plain_faces(cell, face, lane);
3072 if (face_index == numbers::invalid_unsigned_int)
3073 return numbers::invalid_unsigned_int;
3074
3075 unsigned int cell_neighbor = face_info.faces[face_index / n_lanes]
3076 .cells_interior[face_index % n_lanes];
3077 if (cell_neighbor == cell_this)
3078 cell_neighbor = face_info.faces[face_index / n_lanes]
3079 .cells_exterior[face_index % n_lanes];
3080 return cell_neighbor;
3081 };
3082
3083 faces_by_cells_type.resize(cell_type.size());
3084 for (unsigned int cell = 0; cell < cell_type.size(); ++cell)
3085 for (const unsigned int face : GeometryInfo<dim>::face_indices())
3086 {
3087 faces_by_cells_type[cell][face] = cell_type[cell];
3088 for (unsigned int v = 0; v < n_lanes; ++v)
3089 {
3090 const unsigned int cell_neighbor =
3091 compute_neighbor_index(cell, face, v);
3092 if (cell_neighbor != numbers::invalid_unsigned_int)
3093 faces_by_cells_type[cell][face] =
3094 std::max(faces_by_cells_type[cell][face],
3095 cell_type[cell_neighbor / n_lanes]);
3096 }
3097 }
3098
3099 for (unsigned int my_q = 0; my_q < n_quads; ++my_q)
3100 {
3101 // since we already know the cell type, we can pre-allocate the right
3102 // amount of data straight away and we just need to do some basic
3103 // counting
3104 AssertDimension(cell_type.size(), cells.size() / n_lanes);
3105 face_data_by_cells[my_q].data_index_offsets.resize(
3106 cell_type.size() * ReferenceCells::max_n_faces<dim>());
3107 if (update_flags & update_quadrature_points)
3108 face_data_by_cells[my_q].quadrature_point_offsets.resize(
3109 cell_type.size() * ReferenceCells::max_n_faces<dim>());
3110 std::size_t storage_length = 0;
3111 for (unsigned int i = 0; i < cell_type.size(); ++i)
3112 for (const unsigned int face : GeometryInfo<dim>::face_indices())
3113 {
3114 if (faces_by_cells_type[i][face] <= affine)
3115 {
3116 face_data_by_cells[my_q].data_index_offsets
3117 [i * ReferenceCells::max_n_faces<dim>() + face] =
3118 storage_length;
3119 ++storage_length;
3120 }
3121 else
3122 {
3123 face_data_by_cells[my_q].data_index_offsets
3124 [i * ReferenceCells::max_n_faces<dim>() + face] =
3125 storage_length;
3126 storage_length +=
3127 face_data_by_cells[my_q].descriptor[0].n_q_points;
3128 }
3129 if (update_flags & update_quadrature_points)
3130 face_data_by_cells[my_q].quadrature_point_offsets
3131 [i * ReferenceCells::max_n_faces<dim>() + face] =
3132 (i * ReferenceCells::max_n_faces<dim>() + face) *
3133 face_data_by_cells[my_q].descriptor[0].n_q_points;
3134 }
3135 face_data_by_cells[my_q].JxW_values.resize_fast(
3136 storage_length * ReferenceCells::max_n_faces<dim>());
3137 face_data_by_cells[my_q].jacobians[0].resize_fast(
3138 storage_length * ReferenceCells::max_n_faces<dim>());
3139 face_data_by_cells[my_q].jacobians[1].resize_fast(
3140 storage_length * ReferenceCells::max_n_faces<dim>());
3141 if (update_flags & update_normal_vectors)
3142 face_data_by_cells[my_q].normal_vectors.resize_fast(
3143 storage_length * ReferenceCells::max_n_faces<dim>());
3144 if (update_flags & update_normal_vectors &&
3145 update_flags & update_jacobians)
3146 face_data_by_cells[my_q].normals_times_jacobians[0].resize_fast(
3147 storage_length * ReferenceCells::max_n_faces<dim>());
3148 if (update_flags & update_normal_vectors &&
3149 update_flags & update_jacobians)
3150 face_data_by_cells[my_q].normals_times_jacobians[1].resize_fast(
3151 storage_length * ReferenceCells::max_n_faces<dim>());
3152 if (update_flags & update_jacobian_grads)
3153 {
3154 face_data_by_cells[my_q].jacobian_gradients[0].resize_fast(
3155 storage_length * ReferenceCells::max_n_faces<dim>());
3156 face_data_by_cells[my_q]
3157 .jacobian_gradients_non_inverse[0]
3158 .resize_fast(storage_length *
3159 ReferenceCells::max_n_faces<dim>());
3160 }
3161
3162 if (update_flags & update_quadrature_points)
3163 face_data_by_cells[my_q].quadrature_points.resize_fast(
3164 cell_type.size() * ReferenceCells::max_n_faces<dim>() *
3165 face_data_by_cells[my_q].descriptor[0].n_q_points);
3166 }
3167
3168 FE_Nothing<dim> dummy_fe;
3169 // currently no hp-indices implemented
3170 const unsigned int fe_index = 0;
3171 std::vector<std::vector<std::shared_ptr<dealii::FEFaceValues<dim>>>>
3172 fe_face_values(face_data_by_cells.size());
3173 for (unsigned int i = 0; i < fe_face_values.size(); ++i)
3174 fe_face_values[i].resize(face_data_by_cells[i].descriptor.size());
3175 std::vector<std::vector<std::shared_ptr<dealii::FEFaceValues<dim>>>>
3176 fe_face_values_neigh(face_data_by_cells.size());
3177 for (unsigned int i = 0; i < fe_face_values_neigh.size(); ++i)
3178 fe_face_values_neigh[i].resize(face_data_by_cells[i].descriptor.size());
3179 for (unsigned int cell = 0; cell < cell_type.size(); ++cell)
3180 for (unsigned int my_q = 0; my_q < face_data_by_cells.size(); ++my_q)
3181 for (const unsigned int face : GeometryInfo<dim>::face_indices())
3182 {
3183 if (fe_face_values[my_q][fe_index].get() == nullptr)
3184 fe_face_values[my_q][fe_index] =
3185 std::make_shared<dealii::FEFaceValues<dim>>(
3186 mapping,
3187 dummy_fe,
3188 face_data[my_q].q_collection[fe_index],
3189 update_flags);
3190 if (fe_face_values_neigh[my_q][fe_index].get() == nullptr)
3191 fe_face_values_neigh[my_q][fe_index] =
3192 std::make_shared<dealii::FEFaceValues<dim>>(
3193 mapping,
3194 dummy_fe,
3195 face_data[my_q].q_collection[fe_index],
3196 update_flags);
3197 dealii::FEFaceValues<dim> &fe_val =
3198 *fe_face_values[my_q][fe_index];
3199 dealii::FEFaceValues<dim> &fe_val_neigh =
3200 *fe_face_values_neigh[my_q][fe_index];
3201 const unsigned int offset =
3202 face_data_by_cells[my_q].data_index_offsets
3203 [cell * ReferenceCells::max_n_faces<dim>() + face];
3204
3205 const GeometryType my_cell_type = faces_by_cells_type[cell][face];
3206
3207 for (unsigned int v = 0; v < n_lanes; ++v)
3208 {
3209 typename dealii::Triangulation<dim>::cell_iterator cell_it(
3210 &tria,
3211 cells[cell * n_lanes + v].first,
3212 cells[cell * n_lanes + v].second);
3213 fe_val.reinit(cell_it, face);
3214
3215 const unsigned int cell_neighbor =
3216 compute_neighbor_index(cell, face, v);
3217
3218 if (cell_neighbor != numbers::invalid_unsigned_int)
3219 {
3220 typename dealii::Triangulation<dim>::cell_iterator
3221 cell_it_neigh(&tria,
3222 cells[cell_neighbor].first,
3223 cells[cell_neighbor].second);
3224 fe_val_neigh.reinit(cell_it_neigh,
3225 cell_it->at_boundary(face) ?
3226 cell_it->periodic_neighbor_face_no(
3227 face) :
3228 cell_it->neighbor_face_no(face));
3229 }
3230
3231 // copy data for affine data type
3232 if (my_cell_type <= affine)
3233 {
3234 if (update_flags & update_JxW_values)
3235 face_data_by_cells[my_q].JxW_values[offset][v] =
3236 fe_val.JxW(0) / fe_val.get_quadrature().weight(0);
3237 if (update_flags & update_jacobians)
3238 {
3239 DerivativeForm<1, dim, dim> inv_jac =
3240 fe_val.jacobian(0).covariant_form();
3241 for (unsigned int d = 0; d < dim; ++d)
3242 for (unsigned int e = 0; e < dim; ++e)
3243 {
3244 const unsigned int ee = ExtractFaceHelper::
3245 reorder_face_derivative_indices<dim>(face, e);
3246 face_data_by_cells[my_q]
3247 .jacobians[0][offset][d][e][v] =
3248 inv_jac[d][ee];
3249 }
3250 }
3251 if (cell_neighbor != numbers::invalid_unsigned_int &&
3252 (update_flags & update_jacobians))
3253 for (unsigned int q = 0; q < fe_val.n_quadrature_points;
3254 ++q)
3255 {
3256 DerivativeForm<1, dim, dim> inv_jac =
3257 fe_val_neigh.jacobian(q).covariant_form();
3258 for (unsigned int d = 0; d < dim; ++d)
3259 for (unsigned int e = 0; e < dim; ++e)
3260 {
3261 const unsigned int ee = ExtractFaceHelper::
3262 reorder_face_derivative_indices<dim>(
3263 fe_val_neigh.get_face_number(), e);
3264 face_data_by_cells[my_q]
3265 .jacobians[1][offset][d][e][v] =
3266 inv_jac[d][ee];
3267 }
3268 }
3269 if (update_flags & update_jacobian_grads)
3270 {
3271 DEAL_II_NOT_IMPLEMENTED();
3272 }
3273 if (update_flags & update_normal_vectors)
3274 for (unsigned int d = 0; d < dim; ++d)
3275 face_data_by_cells[my_q]
3276 .normal_vectors[offset][d][v] =
3277 fe_val.normal_vector(0)[d];
3278 }
3279 // copy data for general data type
3280 else
3281 {
3282 if (update_flags & update_JxW_values)
3283 for (unsigned int q = 0; q < fe_val.n_quadrature_points;
3284 ++q)
3285 face_data_by_cells[my_q].JxW_values[offset + q][v] =
3286 fe_val.JxW(q);
3287 if (update_flags & update_jacobians)
3288 for (unsigned int q = 0; q < fe_val.n_quadrature_points;
3289 ++q)
3290 {
3291 DerivativeForm<1, dim, dim> inv_jac =
3292 fe_val.jacobian(q).covariant_form();
3293 for (unsigned int d = 0; d < dim; ++d)
3294 for (unsigned int e = 0; e < dim; ++e)
3295 {
3296 const unsigned int ee = ExtractFaceHelper::
3297 reorder_face_derivative_indices<dim>(face,
3298 e);
3299 face_data_by_cells[my_q]
3300 .jacobians[0][offset + q][d][e][v] =
3301 inv_jac[d][ee];
3302 }
3303 }
3304 if (cell_neighbor != numbers::invalid_unsigned_int &&
3305 (update_flags & update_jacobians))
3306 for (unsigned int q = 0; q < fe_val.n_quadrature_points;
3307 ++q)
3308 {
3309 DerivativeForm<1, dim, dim> inv_jac =
3310 fe_val_neigh.jacobian(q).covariant_form();
3311 for (unsigned int d = 0; d < dim; ++d)
3312 for (unsigned int e = 0; e < dim; ++e)
3313 {
3314 const unsigned int ee = ExtractFaceHelper::
3315 reorder_face_derivative_indices<dim>(
3316 fe_val_neigh.get_face_number(), e);
3317 face_data_by_cells[my_q]
3318 .jacobians[1][offset + q][d][e][v] =
3319 inv_jac[d][ee];
3320 }
3321 }
3322 if (update_flags & update_jacobian_grads)
3323 {
3324 DEAL_II_NOT_IMPLEMENTED();
3325 }
3326 if (update_flags & update_normal_vectors)
3327 for (unsigned int q = 0; q < fe_val.n_quadrature_points;
3328 ++q)
3329 for (unsigned int d = 0; d < dim; ++d)
3330 face_data_by_cells[my_q]
3331 .normal_vectors[offset + q][d][v] =
3332 fe_val.normal_vector(q)[d];
3333 }
3334 if (update_flags & update_quadrature_points)
3335 for (unsigned int q = 0; q < fe_val.n_quadrature_points;
3336 ++q)
3337 for (unsigned int d = 0; d < dim; ++d)
3338 face_data_by_cells[my_q].quadrature_points
3339 [face_data_by_cells[my_q].quadrature_point_offsets
3340 [cell * ReferenceCells::max_n_faces<dim>() +
3341 face] +
3342 q][d][v] = fe_val.quadrature_point(q)[d];
3343 }
3344 if (update_flags & update_normal_vectors &&
3345 update_flags & update_jacobians)
3346 for (unsigned int q = 0;
3347 q <
3348 (my_cell_type <= affine ? 1 : fe_val.n_quadrature_points);
3349 ++q)
3350 face_data_by_cells[my_q]
3351 .normals_times_jacobians[0][offset + q] =
3352 face_data_by_cells[my_q].normal_vectors[offset + q] *
3353 face_data_by_cells[my_q].jacobians[0][offset + q];
3354 if (update_flags & update_normal_vectors &&
3355 update_flags & update_jacobians)
3356 for (unsigned int q = 0;
3357 q <
3358 (my_cell_type <= affine ? 1 : fe_val.n_quadrature_points);
3359 ++q)
3360 face_data_by_cells[my_q]
3361 .normals_times_jacobians[1][offset + q] =
3362 face_data_by_cells[my_q].normal_vectors[offset + q] *
3363 face_data_by_cells[my_q].jacobians[1][offset + q];
3364 }
3365 }
3366
3367
3368
3369 template <int dim, typename Number, typename VectorizedArrayType>
3370 std::size_t
3371 MappingInfo<dim, Number, VectorizedArrayType>::memory_consumption() const
3372 {
3373 std::size_t memory = MemoryConsumption::memory_consumption(cell_data);
3374 memory += MemoryConsumption::memory_consumption(face_data);
3375 memory += MemoryConsumption::memory_consumption(face_data_by_cells);
3376 memory += cell_type.capacity() * sizeof(GeometryType);
3377 memory += face_type.capacity() * sizeof(GeometryType);
3378 memory += faces_by_cells_type.capacity() *
3379 ReferenceCells::max_n_faces<dim>() * sizeof(GeometryType);
3380 memory += sizeof(*this);
3381 return memory;
3382 }
3383
3384
3385
3386 template <int dim, typename Number, typename VectorizedArrayType>
3387 template <typename StreamType>
3388 void
3389 MappingInfo<dim, Number, VectorizedArrayType>::print_memory_consumption(
3390 StreamType &out,
3391 const TaskInfo &task_info) const
3392 {
3393 out << " Cell types: ";
3394 task_info.print_memory_statistics(out,
3395 cell_type.capacity() *
3396 sizeof(GeometryType));
3397 out << " Face types: ";
3398 task_info.print_memory_statistics(out,
3399 face_type.capacity() *
3400 sizeof(GeometryType));
3401
3402 out << " Faces by cells types: ";
3403 task_info.print_memory_statistics(out,
3404 faces_by_cells_type.capacity() *
3405 ReferenceCells::max_n_faces<dim>() *
3406 sizeof(GeometryType));
3407
3408 for (unsigned int j = 0; j < cell_data.size(); ++j)
3409 {
3410 out << " Data component " << j << std::endl;
3411 cell_data[j].print_memory_consumption(out, task_info);
3412 face_data[j].print_memory_consumption(out, task_info);
3413 face_data_by_cells[j].print_memory_consumption(out, task_info);
3414 }
3415 }
3416
3417 } // namespace MatrixFreeFunctions
3418 } // end of namespace internal
3419
3420 DEAL_II_NAMESPACE_CLOSE
3421
3422 #endif