| 1 |
// ------------------------------------------------------------------------ |
| 2 |
// |
| 3 |
// SPDX-License-Identifier: LGPL-2.1-or-later |
| 4 |
// Copyright (C) 2015 - 2024 by the deal.II authors |
| 5 |
// |
| 6 |
// This file is part of the deal.II library. |
| 7 |
// |
| 8 |
// Part of the source code is dual licensed under Apache-2.0 WITH |
| 9 |
// LLVM-exception OR LGPL-2.1-or-later. Detailed license information |
| 10 |
// governing the source code and code contributions can be found in |
| 11 |
// LICENSE.md and CONTRIBUTING.md at the top level directory of deal.II. |
| 12 |
// |
| 13 |
// ------------------------------------------------------------------------ |
| 14 |
|
| 15 |
#ifndef dealii_distributed_tria_base_h |
| 16 |
#define dealii_distributed_tria_base_h |
| 17 |
|
| 18 |
|
| 19 |
#include <deal.II/base/config.h> |
| 20 |
|
| 21 |
#include <deal.II/base/enable_observer_pointer.h> |
| 22 |
#include <deal.II/base/mpi_stub.h> |
| 23 |
#include <deal.II/base/observer_pointer.h> |
| 24 |
#include <deal.II/base/partitioner.h> |
| 25 |
#include <deal.II/base/template_constraints.h> |
| 26 |
|
| 27 |
#include <deal.II/grid/tria.h> |
| 28 |
|
| 29 |
#include <functional> |
| 30 |
#include <list> |
| 31 |
#include <set> |
| 32 |
#include <utility> |
| 33 |
#include <vector> |
| 34 |
|
| 35 |
|
| 36 |
DEAL_II_NAMESPACE_OPEN |
| 37 |
|
| 38 |
namespace parallel |
| 39 |
{ |
| 40 |
/** |
| 41 |
* This class describes the interface for all triangulation classes that |
| 42 |
* work in parallel, namely parallel::distributed::Triangulation, |
| 43 |
* parallel::fullydistributed::Triangulation, and |
| 44 |
* parallel::shared::Triangulation. |
| 45 |
* |
| 46 |
* It is, consequently, a class that can be used to test whether a |
| 47 |
* pointer of reference to a triangulation object refers to a |
| 48 |
* sequential triangulation, or whether the triangulation is in fact |
| 49 |
* parallel. In other words, one could write a function like this: |
| 50 |
* @code |
| 51 |
* template <int dim, int spacedim> |
| 52 |
* bool is_parallel (const dealii::Triangulation<dim,spacedim> &tria) |
| 53 |
* { |
| 54 |
* if (dynamic_cast<const parallel::TriangulationBase<dim,spacedim>*> |
| 55 |
* (&tria) |
| 56 |
* != nullptr) |
| 57 |
* return true; |
| 58 |
* else |
| 59 |
* return false; |
| 60 |
* } |
| 61 |
* @endcode |
| 62 |
* |
| 63 |
* All parallel triangulations share certain traits, such as the fact that |
| 64 |
* they communicate via |
| 65 |
* @ref GlossMPICommunicator "MPI communicators" |
| 66 |
* or that they have |
| 67 |
* @ref GlossLocallyOwnedCell "locally owned", |
| 68 |
* @ref GlossGhostCell "ghost", |
| 69 |
* and possibly |
| 70 |
* @ref GlossArtificialCell "artificial cells". |
| 71 |
* This class provides |
| 72 |
* a number of member functions that allows querying some information |
| 73 |
* about the triangulation that is independent of how exactly a |
| 74 |
* parallel triangulation is implemented (i.e., which of the various |
| 75 |
* classes derived from the current one it actually is). |
| 76 |
*/ |
| 77 |
template <int dim, int spacedim = dim> |
| 78 |
DEAL_II_CXX20_REQUIRES((concepts::is_valid_dim_spacedim<dim, spacedim>)) |
| 79 |
class TriangulationBase : public dealii::Triangulation<dim, spacedim> |
| 80 |
{ |
| 81 |
public: |
| 82 |
/** |
| 83 |
* Constructor. |
| 84 |
*/ |
| 85 |
TriangulationBase( |
| 86 |
const MPI_Comm mpi_communicator, |
| 87 |
const typename dealii::Triangulation<dim, spacedim>::MeshSmoothing |
| 88 |
smooth_grid = (dealii::Triangulation<dim, spacedim>::none), |
| 89 |
const bool check_for_distorted_cells = false); |
| 90 |
|
| 91 |
/** |
| 92 |
* Destructor. |
| 93 |
*/ |
| 94 |
virtual ~TriangulationBase() override; |
| 95 |
|
| 96 |
/** |
| 97 |
* Return MPI communicator used by this triangulation. |
| 98 |
*/ |
| 99 |
virtual MPI_Comm |
| 100 |
get_mpi_communicator() const override; |
| 101 |
|
| 102 |
/** |
| 103 |
* Return if multilevel hierarchy is supported and has been constructed. |
| 104 |
*/ |
| 105 |
virtual bool |
| 106 |
is_multilevel_hierarchy_constructed() const = 0; |
| 107 |
|
| 108 |
/** |
| 109 |
* Implementation of the same function as in the base class. |
| 110 |
* |
| 111 |
* @note This function copies the cells, but not the communicator, |
| 112 |
* of the source triangulation. In other words, the resulting |
| 113 |
* triangulation will operate on the communicator it was constructed |
| 114 |
* with. |
| 115 |
*/ |
| 116 |
virtual void |
| 117 |
copy_triangulation( |
| 118 |
const dealii::Triangulation<dim, spacedim> &old_tria) override; |
| 119 |
|
| 120 |
/** |
| 121 |
* Return the number of active cells in the triangulation that are locally |
| 122 |
* owned, i.e. that have a subdomain_id equal to |
| 123 |
* locally_owned_subdomain(). Note that there may be more active cells in |
| 124 |
* the triangulation stored on the present processor, such as for example |
| 125 |
* ghost cells, or cells further away from the locally owned block of |
| 126 |
* cells but that are needed to ensure that the triangulation that stores |
| 127 |
* this processor's set of active cells still remains balanced with |
| 128 |
* respect to the 2:1 size ratio of adjacent cells. |
| 129 |
* |
| 130 |
* As a consequence of the remark above, the result of this function is |
| 131 |
* always smaller or equal to the result of the function with the same |
| 132 |
* name in the ::Triangulation base class, which includes the active ghost |
| 133 |
* and artificial cells (see also |
| 134 |
* @ref GlossArtificialCell |
| 135 |
* and |
| 136 |
* @ref GlossGhostCell). |
| 137 |
*/ |
| 138 |
unsigned int |
| 139 |
n_locally_owned_active_cells() const; |
| 140 |
|
| 141 |
/** |
| 142 |
* Return the sum over all processors of the number of active cells owned |
| 143 |
* by each processor. This equals the overall number of active cells in |
| 144 |
* the triangulation. |
| 145 |
*/ |
| 146 |
virtual types::global_cell_index |
| 147 |
n_global_active_cells() const override; |
| 148 |
|
| 149 |
/** |
| 150 |
* Return the local memory consumption in bytes. |
| 151 |
*/ |
| 152 |
virtual std::size_t |
| 153 |
memory_consumption() const override; |
| 154 |
|
| 155 |
|
| 156 |
/** |
| 157 |
* Return the global maximum level. This may be bigger than the number |
| 158 |
* dealii::Triangulation::n_levels() (a function in this class's base |
| 159 |
* class) returns if the current processor only stores cells in parts of |
| 160 |
* the domain that are not very refined, but if other processors store |
| 161 |
* cells in more deeply refined parts of the domain. |
| 162 |
*/ |
| 163 |
virtual unsigned int |
| 164 |
n_global_levels() const override; |
| 165 |
|
| 166 |
/** |
| 167 |
* Return the subdomain id of those cells that are owned by the current |
| 168 |
* processor. All cells in the triangulation that do not have this |
| 169 |
* subdomain id are either owned by another processor or have children |
| 170 |
* that only exist on other processors. |
| 171 |
*/ |
| 172 |
types::subdomain_id |
| 173 |
locally_owned_subdomain() const override; |
| 174 |
|
| 175 |
/** |
| 176 |
* Return a set of MPI ranks of the processors that have at least one |
| 177 |
* ghost cell adjacent to the cells of the local processor. In other |
| 178 |
* words, this is the set of subdomain_id() for all ghost cells. |
| 179 |
* |
| 180 |
* The returned sets are symmetric, that is if @p i is contained in the |
| 181 |
* list of processor @p j, then @p j will also be contained in the list of |
| 182 |
* processor @p i. |
| 183 |
*/ |
| 184 |
const std::set<types::subdomain_id> & |
| 185 |
ghost_owners() const; |
| 186 |
|
| 187 |
/** |
| 188 |
* Return a set of MPI ranks of the processors that have at least one |
| 189 |
* level ghost cell adjacent to our cells used in geometric multigrid. In |
| 190 |
* other words, this is the set of level_subdomain_id() for all level |
| 191 |
* ghost cells. |
| 192 |
* |
| 193 |
* The returned sets are symmetric, that is if @p i is contained in the |
| 194 |
* list of processor @p j, then @p j will also be contained in the list of |
| 195 |
* processor @p i. |
| 196 |
* |
| 197 |
* @note The level ghost owners can only be determined if the multigrid |
| 198 |
* ownership has been assigned (by setting the |
| 199 |
* construct_multigrid_hierarchy flag at construction time), otherwise the |
| 200 |
* returned set will be empty. |
| 201 |
*/ |
| 202 |
const std::set<types::subdomain_id> & |
| 203 |
level_ghost_owners() const; |
| 204 |
|
| 205 |
std::weak_ptr<const Utilities::MPI::Partitioner> |
| 206 |
global_active_cell_index_partitioner() const override; |
| 207 |
|
| 208 |
std::weak_ptr<const Utilities::MPI::Partitioner> |
| 209 |
global_level_cell_index_partitioner( |
| 210 |
const unsigned int level) const override; |
| 211 |
|
| 212 |
/** |
| 213 |
* Return a vector containing all boundary indicators assigned to boundary |
| 214 |
* faces of active cells of this Triangulation object. Note, that each |
| 215 |
* boundary indicator is reported only once. The size of the return vector |
| 216 |
* will represent the number of different indicators (which is greater or |
| 217 |
* equal one). |
| 218 |
* |
| 219 |
* @see |
| 220 |
* @ref GlossBoundaryIndicator "Glossary entry on boundary indicators" |
| 221 |
* |
| 222 |
* @note This function involves a global communication gathering all current |
| 223 |
* IDs from all processes. |
| 224 |
*/ |
| 225 |
virtual std::vector<types::boundary_id> |
| 226 |
get_boundary_ids() const override; |
| 227 |
|
| 228 |
/** |
| 229 |
* Return a vector containing all manifold indicators assigned to the |
| 230 |
* objects of the active cells of this Triangulation. Note, that each |
| 231 |
* manifold indicator is reported only once. The size of the return vector |
| 232 |
* will represent the number of different indicators (which is greater or |
| 233 |
* equal one). |
| 234 |
* |
| 235 |
* @ingroup manifold |
| 236 |
* |
| 237 |
* @see |
| 238 |
* @ref GlossManifoldIndicator "Glossary entry on manifold indicators" |
| 239 |
* |
| 240 |
* @note This function involves a global communication gathering all current |
| 241 |
* IDs from all processes. |
| 242 |
*/ |
| 243 |
virtual std::vector<types::manifold_id> |
| 244 |
get_manifold_ids() const override; |
| 245 |
|
| 246 |
/** |
| 247 |
* When vertices have been moved locally, for example using code like |
| 248 |
* @code |
| 249 |
* cell->vertex(0) = new_location; |
| 250 |
* @endcode |
| 251 |
* then this function can be used to update the location of vertices |
| 252 |
* between MPI processes. |
| 253 |
* |
| 254 |
* All the vertices that have been moved and might be in the ghost layer |
| 255 |
* of a process have to be reported in the @p vertex_locally_moved |
| 256 |
* argument. This ensures that that part of the information that has to |
| 257 |
* be send between processes is actually sent. Additionally, it is quite |
| 258 |
* important that vertices on the boundary between processes are |
| 259 |
* reported on exactly one process (e.g. the one with the highest id). |
| 260 |
* Otherwise we could expect undesirable results if multiple processes |
| 261 |
* move a vertex differently. A typical strategy is to let processor $i$ |
| 262 |
* move those vertices that are adjacent to cells whose owners include |
| 263 |
* processor $i$ but no other processor $j$ with $j<i$; in other words, |
| 264 |
* for vertices at the boundary of a subdomain, the processor with the |
| 265 |
* lowest subdomain id "owns" a vertex. |
| 266 |
* |
| 267 |
* @note It only makes sense to move vertices that are either located on |
| 268 |
* locally owned cells or on cells in the ghost layer. This is because |
| 269 |
* you can be sure that these vertices indeed exist on the finest mesh |
| 270 |
* aggregated over all processors, whereas vertices on artificial cells |
| 271 |
* but not at least in the ghost layer may or may not exist on the |
| 272 |
* globally finest mesh. Consequently, the @p vertex_locally_moved |
| 273 |
* argument may not contain vertices that aren't at least on ghost |
| 274 |
* cells. |
| 275 |
* |
| 276 |
* @note This function moves vertices in such a way that on every |
| 277 |
* processor, the vertices of every locally owned and ghost cell is |
| 278 |
* consistent with the corresponding location of these cells on other |
| 279 |
* processors. On the other hand, the locations of artificial cells will |
| 280 |
* in general be wrong since artificial cells may or may not exist on |
| 281 |
* other processors and consequently it is not possible to determine |
| 282 |
* their location in any way. This is not usually a problem since one |
| 283 |
* never does anything on artificial cells. However, it may lead to |
| 284 |
* problems if the mesh with moved vertices is refined in a later step. |
| 285 |
* If that's what you want to do, the right way to do it is to save the |
| 286 |
* offset applied to every vertex, call this function, and before |
| 287 |
* refining or coarsening the mesh apply the opposite offset and call |
| 288 |
* this function again. |
| 289 |
* |
| 290 |
* @param vertex_locally_moved A bitmap indicating which vertices have |
| 291 |
* been moved. The size of this array must be equal to |
| 292 |
* Triangulation::n_vertices() and must be a subset of those vertices |
| 293 |
* flagged by GridTools::get_locally_owned_vertices(). |
| 294 |
* |
| 295 |
* @see This function is used, for example, in |
| 296 |
* GridTools::distort_random(). |
| 297 |
*/ |
| 298 |
void |
| 299 |
communicate_locally_moved_vertices( |
| 300 |
const std::vector<bool> &vertex_locally_moved); |
| 301 |
|
| 302 |
virtual types::coarse_cell_id |
| 303 |
n_global_coarse_cells() const override; |
| 304 |
|
| 305 |
/** |
| 306 |
* Reset this triangulation to an empty state by deleting all data. |
| 307 |
* |
| 308 |
* Note that this operation is only allowed if no subscriptions to this |
| 309 |
* object exist any more, such as DoFHandler objects using it. |
| 310 |
*/ |
| 311 |
virtual void |
| 312 |
clear() override; |
| 313 |
|
| 314 |
protected: |
| 315 |
/** |
| 316 |
* MPI communicator to be used for the triangulation. We create a unique |
| 317 |
* communicator for this class, which is a duplicate of the one passed to |
| 318 |
* the constructor. |
| 319 |
*/ |
| 320 |
const MPI_Comm mpi_communicator; |
| 321 |
|
| 322 |
/** |
| 323 |
* The subdomain id to be used for the current processor. This is the MPI |
| 324 |
* rank. |
| 325 |
*/ |
| 326 |
types::subdomain_id my_subdomain; |
| 327 |
|
| 328 |
/** |
| 329 |
* The total number of subdomains (or the size of the MPI communicator). |
| 330 |
*/ |
| 331 |
types::subdomain_id n_subdomains; |
| 332 |
|
| 333 |
/** |
| 334 |
* A structure that contains information about the distributed |
| 335 |
* triangulation. |
| 336 |
*/ |
| 337 |
struct NumberCache |
| 338 |
{ |
| 339 |
/** |
| 340 |
* Number of locally owned active cells of this MPI rank. |
| 341 |
*/ |
| 342 |
unsigned int n_locally_owned_active_cells; |
| 343 |
|
| 344 |
/** |
| 345 |
* The total number of active cells (sum of @p |
| 346 |
* n_locally_owned_active_cells). |
| 347 |
*/ |
| 348 |
types::global_cell_index n_global_active_cells; |
| 349 |
|
| 350 |
/** |
| 351 |
* Number of global coarse cells. |
| 352 |
*/ |
| 353 |
types::coarse_cell_id number_of_global_coarse_cells; |
| 354 |
|
| 355 |
/** |
| 356 |
* The global number of levels computed as the maximum number of levels |
| 357 |
* taken over all MPI ranks, so <tt>n_levels()<=n_global_levels = |
| 358 |
* max(n_levels() on proc i)</tt>. |
| 359 |
*/ |
| 360 |
unsigned int n_global_levels; |
| 361 |
|
| 362 |
/** |
| 363 |
* A set containing the subdomain_id (MPI rank) of the owners of the |
| 364 |
* ghost cells on this processor. |
| 365 |
*/ |
| 366 |
std::set<types::subdomain_id> ghost_owners; |
| 367 |
|
| 368 |
/** |
| 369 |
* A set containing the MPI ranks of the owners of the level ghost cells |
| 370 |
* on this processor (for all levels). |
| 371 |
*/ |
| 372 |
std::set<types::subdomain_id> level_ghost_owners; |
| 373 |
|
| 374 |
/** |
| 375 |
* Partitioner for the global active cell indices. |
| 376 |
*/ |
| 377 |
std::shared_ptr<const Utilities::MPI::Partitioner> |
| 378 |
active_cell_index_partitioner; |
| 379 |
|
| 380 |
/** |
| 381 |
* Partitioner for the global level cell indices for each level. |
| 382 |
*/ |
| 383 |
std::vector<std::shared_ptr<const Utilities::MPI::Partitioner>> |
| 384 |
level_cell_index_partitioners; |
| 385 |
|
| 386 |
NumberCache(); |
| 387 |
}; |
| 388 |
|
| 389 |
NumberCache number_cache; |
| 390 |
|
| 391 |
/** |
| 392 |
* Update the number_cache variable after mesh creation or refinement. |
| 393 |
*/ |
| 394 |
virtual void |
| 395 |
update_number_cache(); |
| 396 |
|
| 397 |
/** |
| 398 |
* Reset global active cell indices and global level cell indices. |
| 399 |
*/ |
| 400 |
void |
| 401 |
reset_global_cell_indices(); |
| 402 |
|
| 403 |
void |
| 404 |
update_reference_cells() override; |
| 405 |
}; |
| 406 |
|
| 407 |
|
| 408 |
/** |
| 409 |
* A base class for distributed triangulations, i.e., triangulations that |
| 410 |
* do not store all cells on all processors. This implies that not |
| 411 |
* every detail of a triangulation may be known on each processor. |
| 412 |
* In particular, you have to expect that triangulations of classes |
| 413 |
* derived from this one only store some of the active cells (namely, |
| 414 |
* the |
| 415 |
* @ref GlossLocallyOwnedCell "locally owned cells"), |
| 416 |
* along with |
| 417 |
* @ref GlossGhostCell "ghost cells" |
| 418 |
* and possibly |
| 419 |
* @ref GlossArtificialCell "artificial cells". |
| 420 |
* In contrast to the classes |
| 421 |
* derived from parallel::TriangulationBase, it is certain that the |
| 422 |
* classes derived from the current class will not store the entire |
| 423 |
* triangulation as long as it has a large enough number of cells. (The |
| 424 |
* difference to parallel::TriangulationBase is that the |
| 425 |
* parallel::shared::Triangulation is derived from |
| 426 |
* parallel::TriangulationBase, but not from the current class.) The |
| 427 |
* distinction is not large in practice: Everything that is difficult for |
| 428 |
* parallel distributed triangulation is generally also difficult for any |
| 429 |
* other kind of parallel triangulation classes; however, this intermediate |
| 430 |
* base class allows to further differentiate between the different kinds of |
| 431 |
* classes providing parallel mesh functionality. |
| 432 |
* |
| 433 |
* This class can, then, be used to test whether a |
| 434 |
* pointer or reference to a triangulation object refers to any kind of |
| 435 |
* parallel triangulation, or whether the triangulation is in fact |
| 436 |
* parallel distributed. In other words, one could write a function like |
| 437 |
* this: |
| 438 |
* @code |
| 439 |
* template <int dim, int spacedim> |
| 440 |
* bool |
| 441 |
* is_parallel_distributed(const dealii::Triangulation<dim,spacedim> &tria) |
| 442 |
* { |
| 443 |
* if(dynamic_cast<const |
| 444 |
* parallel::DistributedTriangulationBase<dim,spacedim>*> |
| 445 |
* (&tria) |
| 446 |
* != nullptr) |
| 447 |
* return true; |
| 448 |
* else |
| 449 |
* return false; |
| 450 |
* } |
| 451 |
* @endcode |
| 452 |
* |
| 453 |
* @dealiiConceptRequires{(concepts::is_valid_dim_spacedim<dim, spacedim>)} |
| 454 |
*/ |
| 455 |
template <int dim, int spacedim = dim> |
| 456 |
DEAL_II_CXX20_REQUIRES((concepts::is_valid_dim_spacedim<dim, spacedim>)) |
| 457 |
class DistributedTriangulationBase |
| 458 |
: public dealii::parallel::TriangulationBase<dim, spacedim> |
| 459 |
{ |
| 460 |
public: |
| 461 |
/** |
| 462 |
* Constructor. |
| 463 |
*/ |
| 464 |
DistributedTriangulationBase( |
| 465 |
const MPI_Comm mpi_communicator, |
| 466 |
const typename dealii::Triangulation<dim, spacedim>::MeshSmoothing |
| 467 |
smooth_grid = (dealii::Triangulation<dim, spacedim>::none), |
| 468 |
const bool check_for_distorted_cells = false); |
| 469 |
|
| 470 |
using cell_iterator = |
| 471 |
typename dealii::Triangulation<dim, spacedim>::cell_iterator; |
| 472 |
|
| 473 |
/** |
| 474 |
* Return true if the triangulation has hanging nodes. |
| 475 |
* |
| 476 |
* In the context of parallel distributed triangulations, every |
| 477 |
* processor stores only that part of the triangulation it owns locally. |
| 478 |
* However, it also stores coarser levels, and to guarantee the |
| 479 |
* 2:1 relationship between cells, this may mean that there are hanging |
| 480 |
* nodes between cells that are not locally owned or ghost cells (i.e., |
| 481 |
* between ghost cells and artificial cells, or between artificial and |
| 482 |
* artificial cells; see |
| 483 |
* @ref GlossArtificialCell "the glossary"). |
| 484 |
* One is not typically interested in this case, so the function returns |
| 485 |
* whether there are hanging nodes between any two cells of the "global" |
| 486 |
* mesh, i.e., the union of locally owned cells on all processors. |
| 487 |
*/ |
| 488 |
virtual bool |
| 489 |
has_hanging_nodes() const override; |
| 490 |
|
| 491 |
/** |
| 492 |
* Save the triangulation into the given file. This file needs to be |
| 493 |
* reachable from all nodes in the computation on a shared network file |
| 494 |
* system. See the SolutionTransfer class on how to store solution vectors |
| 495 |
* into this file. Additional cell-based data can be saved using |
| 496 |
* register_data_attach(). |
| 497 |
*/ |
| 498 |
using Triangulation<dim, spacedim>::save; |
| 499 |
|
| 500 |
|
| 501 |
/** |
| 502 |
* Load the triangulation saved with save() back in. Cell-based data that |
| 503 |
* was saved with register_data_attach() can be read in with |
| 504 |
* notify_ready_to_unpack() after calling Triangulation<dim, |
| 505 |
* spacedim>::load. |
| 506 |
*/ |
| 507 |
using Triangulation<dim, spacedim>::load; |
| 508 |
}; |
| 509 |
|
| 510 |
} // namespace parallel |
| 511 |
|
| 512 |
DEAL_II_NAMESPACE_CLOSE |
| 513 |
|
| 514 |
#endif |