| 1 |
// ------------------------------------------------------------------------ |
| 2 |
// |
| 3 |
// SPDX-License-Identifier: LGPL-2.1-or-later |
| 4 |
// Copyright (C) 2021 - 2025 by the deal.II authors |
| 5 |
// |
| 6 |
// This file is part of the deal.II library. |
| 7 |
// |
| 8 |
// Part of the source code is dual licensed under Apache-2.0 WITH |
| 9 |
// LLVM-exception OR LGPL-2.1-or-later. Detailed license information |
| 10 |
// governing the source code and code contributions can be found in |
| 11 |
// LICENSE.md and CONTRIBUTING.md at the top level directory of deal.II. |
| 12 |
// |
| 13 |
// ------------------------------------------------------------------------ |
| 14 |
|
| 15 |
#ifndef dealii_mpi_mpi_remote_point_evaluation_h |
| 16 |
#define dealii_mpi_mpi_remote_point_evaluation_h |
| 17 |
|
| 18 |
#include <deal.II/base/config.h> |
| 19 |
|
| 20 |
#include <deal.II/base/mpi.h> |
| 21 |
#include <deal.II/base/mpi_tags.h> |
| 22 |
|
| 23 |
#include <deal.II/dofs/dof_handler.h> |
| 24 |
|
| 25 |
#include <boost/signals2/connection.hpp> |
| 26 |
|
| 27 |
|
| 28 |
DEAL_II_NAMESPACE_OPEN |
| 29 |
|
| 30 |
// Forward declarations |
| 31 |
namespace GridTools |
| 32 |
{ |
| 33 |
template <int dim, int spacedim> |
| 34 |
class Cache; |
| 35 |
|
| 36 |
namespace internal |
| 37 |
{ |
| 38 |
template <int dim, int spacedim> |
| 39 |
struct DistributedComputePointLocationsInternal; |
| 40 |
} |
| 41 |
} // namespace GridTools |
| 42 |
|
| 43 |
namespace Utilities |
| 44 |
{ |
| 45 |
namespace MPI |
| 46 |
{ |
| 47 |
/** |
| 48 |
* @brief Communicate values between a mesh and arbitrary points |
| 49 |
* |
| 50 |
* This class enables evaluation of finite element functions defined |
| 51 |
* on a Triangulation at arbitrary points inside the mesh. The underlying |
| 52 |
* finite element mesh can be distributed among processes, which makes |
| 53 |
* the operations more involved due to communication. |
| 54 |
* |
| 55 |
* Once initialized, the data exchange is handled by the functions |
| 56 |
* process_and_evaluate(), that transfers data from the Triangulation |
| 57 |
* to the set of points, and evaluate_and_process(), that performs |
| 58 |
* the inverse operation (taking data defined at each point and making |
| 59 |
* it available in the corresponding cell of the Triangulation). |
| 60 |
* |
| 61 |
* The helper functions VectorTools::point_values() and |
| 62 |
* VectorTools::point_gradients (the overloads taking an instance |
| 63 |
* of RemotePointEvaluation) can be used in place of |
| 64 |
* process_and_evaluate() and perform the evaluation of a finite |
| 65 |
* element function. |
| 66 |
* |
| 67 |
* The template argument `DataType` of the process_and_evaluate() and |
| 68 |
* evaluate_and_process() functions can be a scalar (`float`, `double`), |
| 69 |
* `Tensor<rank,dim>`, or `Point<dim>`. Additionally, one can specify |
| 70 |
* the number of components (in case `DataType` is a scalar). |
| 71 |
* |
| 72 |
* @note Usage and implementation details of this class and the |
| 73 |
* helper functions above are explained in detail in step-87. |
| 74 |
* |
| 75 |
* @note The name of the fields are chosen with the method |
| 76 |
* evaluate_and_process() in mind. Here, quantities are |
| 77 |
* computed at specified arbitrary positioned points (and even on remote |
| 78 |
* processes in the MPI universe) cell by cell and these values are sent |
| 79 |
* to requesting processes, which receive the result and resort the |
| 80 |
* result according to the points. |
| 81 |
*/ |
| 82 |
template <int dim, int spacedim = dim> |
| 83 |
class RemotePointEvaluation |
| 84 |
{ |
| 85 |
public: |
| 86 |
/** |
| 87 |
* AdditionalData structure that can be used to tweak parameters |
| 88 |
* of RemotePointEvaluation. |
| 89 |
*/ |
| 90 |
struct AdditionalData |
| 91 |
{ |
| 92 |
public: |
| 93 |
/** |
| 94 |
* Constructor. |
| 95 |
*/ |
| 96 |
AdditionalData( |
| 97 |
const double tolerance = 1e-6, |
| 98 |
const bool enforce_unique_mapping = false, |
| 99 |
const unsigned int rtree_level = 0, |
| 100 |
const std::function<std::vector<bool>()> &marked_vertices = {}); |
| 101 |
|
| 102 |
/** |
| 103 |
* Tolerance in terms of unit cell coordinates for determining all cells |
| 104 |
* around a point passed to RemotePointEvaluation during reinit(). |
| 105 |
* Depending on the problem, it might be necessary to adjust the |
| 106 |
* tolerance in order to be able to identify a cell. Floating point |
| 107 |
* arithmetic implies that a point will, in general, not lie exactly on |
| 108 |
* a vertex, edge, or face. |
| 109 |
*/ |
| 110 |
double tolerance; |
| 111 |
|
| 112 |
/** |
| 113 |
* Enforce unique mapping, i.e., (one-to-one) relation of points and |
| 114 |
* cells. |
| 115 |
*/ |
| 116 |
bool enforce_unique_mapping; |
| 117 |
|
| 118 |
/** |
| 119 |
* RTree level to be used during the construction of the bounding boxes. |
| 120 |
*/ |
| 121 |
unsigned int rtree_level; |
| 122 |
|
| 123 |
/** |
| 124 |
* Function that marks relevant vertices to make search of active cells |
| 125 |
* around point more efficient. |
| 126 |
*/ |
| 127 |
std::function<std::vector<bool>()> marked_vertices; |
| 128 |
}; |
| 129 |
|
| 130 |
/** |
| 131 |
* Constructor. |
| 132 |
* |
| 133 |
* @param additional_data Configure options for RemotePointEvaluation. |
| 134 |
*/ |
| 135 |
RemotePointEvaluation( |
| 136 |
const AdditionalData &additional_data = AdditionalData()); |
| 137 |
|
| 138 |
/** |
| 139 |
* Constructor. This constructor is deprecated. Use the other constructor |
| 140 |
* taking AdditionalData instead. |
| 141 |
* |
| 142 |
* @param tolerance Tolerance in terms of unit cell coordinates for |
| 143 |
* determining all cells around a point passed to the class during |
| 144 |
* reinit(). Depending on the problem, it might be necessary to adjust |
| 145 |
* the tolerance in order to be able to identify a cell. |
| 146 |
* Floating point arithmetic implies that a point will, in general, not |
| 147 |
* lie exactly on a vertex, edge, or face. |
| 148 |
* @param enforce_unique_mapping Enforce unique mapping, i.e., |
| 149 |
* (one-to-one) relation of points and cells. |
| 150 |
* @param rtree_level RTree level to be used during the construction of the bounding boxes. |
| 151 |
* @param marked_vertices Function that marks relevant vertices to make search |
| 152 |
* of active cells around point more efficient. |
| 153 |
* |
| 154 |
* @deprecated |
| 155 |
*/ |
| 156 |
DEAL_II_DEPRECATED_WITH_COMMENT( |
| 157 |
"Use the constructor with AdditionalData struct.") |
| 158 |
RemotePointEvaluation( |
| 159 |
const double tolerance, |
| 160 |
const bool enforce_unique_mapping = false, |
| 161 |
const unsigned int rtree_level = 0, |
| 162 |
const std::function<std::vector<bool>()> &marked_vertices = {}); |
| 163 |
|
| 164 |
/** |
| 165 |
* Destructor. |
| 166 |
*/ |
| 167 |
~RemotePointEvaluation(); |
| 168 |
|
| 169 |
/** |
| 170 |
* Set up internal data structures and communication pattern based on |
| 171 |
* a list of points @p points and mesh description (@p tria and @p |
| 172 |
* mapping). |
| 173 |
* |
| 174 |
* @warning This is a collective call that needs to be executed by all |
| 175 |
* processors in the communicator. |
| 176 |
* |
| 177 |
* @note If you want to be sure that all points have been found, call |
| 178 |
* all_points_found() after calling this function. |
| 179 |
*/ |
| 180 |
void |
| 181 |
reinit(const std::vector<Point<spacedim>> &points, |
| 182 |
const Triangulation<dim, spacedim> &tria, |
| 183 |
const Mapping<dim, spacedim> &mapping); |
| 184 |
|
| 185 |
/** |
| 186 |
* Set up internal data structures and communication pattern based on |
| 187 |
* an existing Cache and a list of points @p points. |
| 188 |
* |
| 189 |
* @warning This is a collective call that needs to be executed by all |
| 190 |
* processors in the communicator. |
| 191 |
* |
| 192 |
* @note If you want to be sure that all points have been found, call |
| 193 |
* all_points_found() after calling this function. |
| 194 |
*/ |
| 195 |
void |
| 196 |
reinit(const GridTools::Cache<dim, spacedim> &cache, |
| 197 |
const std::vector<Point<spacedim>> &points); |
| 198 |
|
| 199 |
/** |
| 200 |
* Set up internal data structures and communication pattern based on |
| 201 |
* GridTools::internal::DistributedComputePointLocationsInternal. |
| 202 |
* |
| 203 |
* This function is called internally by the reinit() function above. |
| 204 |
* Having it as a separate function makes it possible to set up the class |
| 205 |
* if it is known in which cells corresponding reference points are |
| 206 |
* located (e.g. if intersections of cells are known). |
| 207 |
*/ |
| 208 |
void |
| 209 |
reinit(const GridTools::internal:: |
| 210 |
DistributedComputePointLocationsInternal<dim, spacedim> &data, |
| 211 |
const Triangulation<dim, spacedim> &tria, |
| 212 |
const Mapping<dim, spacedim> &mapping); |
| 213 |
|
| 214 |
/** |
| 215 |
* Helper class to store and to access data of points positioned in |
| 216 |
* processed cells. |
| 217 |
*/ |
| 218 |
class CellData |
| 219 |
{ |
| 220 |
public: |
| 221 |
/** |
| 222 |
* Constructor. |
| 223 |
* |
| 224 |
* @param triangulation Triangulation of the domain. |
| 225 |
*/ |
| 226 |
CellData(const Triangulation<dim, spacedim> &triangulation); |
| 227 |
|
| 228 |
/** |
| 229 |
* Return an object that can be thought of as an array containing all |
| 230 |
* indices from zero (inclusive) to the total number of cells |
| 231 |
* where points reside given by `cells.size()` (exclusive). This allows |
| 232 |
* one to write code using range-based `for` loops of the following |
| 233 |
* kind: |
| 234 |
* |
| 235 |
* @code |
| 236 |
* for (const auto cell : cell_data.cell_indices()) |
| 237 |
* { |
| 238 |
* const auto cell_dofs = |
| 239 |
* cell_data.get_active_cell_iterator(cell)->as_dof_handler_iterator( |
| 240 |
* dof_handler); |
| 241 |
* |
| 242 |
* const auto unit_points = cell_data.get_unit_points(cell); |
| 243 |
* const auto local_value = cell_data.get_data_view(cell, values); |
| 244 |
* |
| 245 |
* // user code: not shown |
| 246 |
* } |
| 247 |
* @endcode |
| 248 |
* |
| 249 |
* Here, we are looping over all cells where points reside and |
| 250 |
* use the index to call the functions get_active_cell_iterator(), |
| 251 |
* get_unit_points(), and get_data_view(). |
| 252 |
*/ |
| 253 |
std_cxx20::ranges::iota_view<unsigned int, unsigned int> |
| 254 |
cell_indices() const; |
| 255 |
|
| 256 |
/** |
| 257 |
* Return active cell iterator of the processed cell @p cell. |
| 258 |
*/ |
| 259 |
typename Triangulation<dim, spacedim>::active_cell_iterator |
| 260 |
get_active_cell_iterator(const unsigned int cell) const; |
| 261 |
|
| 262 |
/** |
| 263 |
* Return unit points of the processed cell @p cell. |
| 264 |
*/ |
| 265 |
ArrayView<const Point<dim>> |
| 266 |
get_unit_points(const unsigned int cell) const; |
| 267 |
|
| 268 |
/** |
| 269 |
* Return local view of the processed cell @p cell for the vector @p values. |
| 270 |
*/ |
| 271 |
template <typename DataType> |
| 272 |
ArrayView<DataType> |
| 273 |
get_data_view(const unsigned int cell, |
| 274 |
const ArrayView<DataType> &values) const; |
| 275 |
|
| 276 |
/** |
| 277 |
* Level and index of processed cells. |
| 278 |
*/ |
| 279 |
std::vector<std::pair<int, int>> cells; |
| 280 |
|
| 281 |
/** |
| 282 |
* Pointers to the start and end of the (reference) points |
| 283 |
* associated to the cell. |
| 284 |
*/ |
| 285 |
std::vector<unsigned int> reference_point_ptrs; |
| 286 |
|
| 287 |
/** |
| 288 |
* Reference points in the interval [0,1]^dim. |
| 289 |
*/ |
| 290 |
std::vector<Point<dim>> reference_point_values; |
| 291 |
|
| 292 |
private: |
| 293 |
/** |
| 294 |
* Reference to the underlying triangulation needed for |
| 295 |
* get_active_cell_iterator(). |
| 296 |
*/ |
| 297 |
const Triangulation<dim, spacedim> &triangulation; |
| 298 |
}; |
| 299 |
|
| 300 |
/** |
| 301 |
* Return internal data structure storing the data of points |
| 302 |
* positioned in cells. |
| 303 |
*/ |
| 304 |
const CellData & |
| 305 |
get_cell_data() const; |
| 306 |
|
| 307 |
/** |
| 308 |
* Evaluate function @p evaluation_function in the given points and |
| 309 |
* triangulation. The result is stored in @p output. |
| 310 |
* |
| 311 |
* @p buffer is a temporary buffer than can be used to avoid extra |
| 312 |
* allocations. |
| 313 |
* |
| 314 |
* @note If the map of points to cells is not a |
| 315 |
* one-to-one relation (is_map_unique()==false), the result needs to be |
| 316 |
* processed with the help of get_point_ptrs(). This |
| 317 |
* might be the case if a point coincides with a geometric entity (e.g., |
| 318 |
* vertex) that is shared by multiple cells or a point is outside of the |
| 319 |
* computational domain. |
| 320 |
* |
| 321 |
* @warning This is a collective call that needs to be executed by all |
| 322 |
* processors in the communicator. |
| 323 |
* |
| 324 |
* @note This function can handle arbitrary types including scalar and |
| 325 |
* Tensor objects. In the case that scalar types are used, one can |
| 326 |
* specify the number of components @p n_components. This allows to |
| 327 |
* provide unrolled tensors, which is useful, e.g., if its dimension |
| 328 |
* and its rank is not known at compile time. |
| 329 |
*/ |
| 330 |
template <typename DataType, unsigned int n_components = 1> |
| 331 |
void |
| 332 |
evaluate_and_process( |
| 333 |
std::vector<DataType> &output, |
| 334 |
std::vector<DataType> &buffer, |
| 335 |
const std::function<void(const ArrayView<DataType> &, const CellData &)> |
| 336 |
&evaluation_function, |
| 337 |
const bool sort_data = true) const; |
| 338 |
|
| 339 |
/** |
| 340 |
* Same as above but with the result provided as return value and |
| 341 |
* without external allocation of a user-provided buffer. |
| 342 |
*/ |
| 343 |
template <typename DataType, unsigned int n_components = 1> |
| 344 |
std::vector<DataType> |
| 345 |
evaluate_and_process( |
| 346 |
const std::function<void(const ArrayView<DataType> &, const CellData &)> |
| 347 |
&evaluation_function, |
| 348 |
const bool sort_data = true) const; |
| 349 |
|
| 350 |
/** |
| 351 |
* This method is the inverse of the method evaluate_and_process(). It |
| 352 |
* makes the data at the points, provided by @p input, available in the |
| 353 |
* function @p evaluation_function. |
| 354 |
* |
| 355 |
* @warning This is a collective call that needs to be executed by all |
| 356 |
* processors in the communicator. |
| 357 |
* |
| 358 |
* @note This function can handle arbitrary types including scalar and |
| 359 |
* Tensor objects. In the case that scalar types are used, one can |
| 360 |
* specify the number of components @p n_components. This allows to |
| 361 |
* provide unrolled tensors, which is useful, e.g., if its dimension |
| 362 |
* and its rank is not known at compile time. |
| 363 |
*/ |
| 364 |
template <typename DataType, unsigned int n_components = 1> |
| 365 |
void |
| 366 |
process_and_evaluate( |
| 367 |
const std::vector<DataType> &input, |
| 368 |
std::vector<DataType> &buffer, |
| 369 |
const std::function<void(const ArrayView<const DataType> &, |
| 370 |
const CellData &)> &evaluation_function, |
| 371 |
const bool sort_data = true) const; |
| 372 |
|
| 373 |
/** |
| 374 |
* Same as above but without external allocation of a user-provided |
| 375 |
* buffer. |
| 376 |
*/ |
| 377 |
template <typename DataType, unsigned int n_components = 1> |
| 378 |
void |
| 379 |
process_and_evaluate( |
| 380 |
const std::vector<DataType> &input, |
| 381 |
const std::function<void(const ArrayView<const DataType> &, |
| 382 |
const CellData &)> &evaluation_function, |
| 383 |
const bool sort_data = true) const; |
| 384 |
|
| 385 |
/** |
| 386 |
* Return a CRS-like data structure to determine the position of the |
| 387 |
* result corresponding a point and the amount. |
| 388 |
*/ |
| 389 |
const std::vector<unsigned int> & |
| 390 |
get_point_ptrs() const; |
| 391 |
|
| 392 |
/** |
| 393 |
* Return if points and cells have a one-to-one relation. This is not the |
| 394 |
* case if a point is not owned by any cell (the point is outside of the |
| 395 |
* domain) or if multiple cells own the point (the point is positioned |
| 396 |
* on a geometric entity shared by neighboring cells). |
| 397 |
*/ |
| 398 |
bool |
| 399 |
is_map_unique() const; |
| 400 |
|
| 401 |
/** |
| 402 |
* Return if all points could be found in the domain. |
| 403 |
*/ |
| 404 |
bool |
| 405 |
all_points_found() const; |
| 406 |
|
| 407 |
/** |
| 408 |
* Return if point @p i could be found in the domain. |
| 409 |
*/ |
| 410 |
bool |
| 411 |
point_found(const unsigned int i) const; |
| 412 |
|
| 413 |
/** |
| 414 |
* Return the Triangulation object used during reinit(). |
| 415 |
*/ |
| 416 |
const Triangulation<dim, spacedim> & |
| 417 |
get_triangulation() const; |
| 418 |
|
| 419 |
/** |
| 420 |
* Return the Mapping object used during reinit(). |
| 421 |
*/ |
| 422 |
const Mapping<dim, spacedim> & |
| 423 |
get_mapping() const; |
| 424 |
|
| 425 |
/** |
| 426 |
* Return if the internal data structures have been set up and if yes |
| 427 |
* whether they are still valid (and not invalidated due to changes of the |
| 428 |
* Triangulation). |
| 429 |
*/ |
| 430 |
bool |
| 431 |
is_ready() const; |
| 432 |
|
| 433 |
/** |
| 434 |
* Return permutation needed for sending. If this is applied, evaluation |
| 435 |
* results do not have to be sorted according to the receiving ranks |
| 436 |
* during evaluate_and_process(). |
| 437 |
*/ |
| 438 |
const std::vector<unsigned int> & |
| 439 |
get_send_permutation() const; |
| 440 |
|
| 441 |
/** |
| 442 |
* Data is contiguous rank by rank directly after MPI communication. |
| 443 |
* If no sorting is requested during evaluate_and_process(), |
| 444 |
* this permutation allows to access all values corresponding to an |
| 445 |
* evaluation point (sorted into different cells). |
| 446 |
*/ |
| 447 |
const std::vector<unsigned int> & |
| 448 |
get_inverse_recv_permutation() const; |
| 449 |
|
| 450 |
|
| 451 |
private: |
| 452 |
/** |
| 453 |
* Additional data with basic settings. |
| 454 |
*/ |
| 455 |
const AdditionalData additional_data; |
| 456 |
|
| 457 |
/** |
| 458 |
* Storage for the status of the triangulation signal. |
| 459 |
*/ |
| 460 |
boost::signals2::connection tria_signal; |
| 461 |
|
| 462 |
/** |
| 463 |
* Flag indicating if the reinit() function has been called and if yes |
| 464 |
* the triangulation has not been modified since then (potentially |
| 465 |
* invalidating the communication pattern). |
| 466 |
*/ |
| 467 |
bool ready_flag; |
| 468 |
|
| 469 |
/** |
| 470 |
* Reference to the Triangulation object used during reinit(). |
| 471 |
*/ |
| 472 |
ObserverPointer<const Triangulation<dim, spacedim>> tria; |
| 473 |
|
| 474 |
/** |
| 475 |
* Reference to the Mapping object used during reinit(). |
| 476 |
*/ |
| 477 |
ObserverPointer<const Mapping<dim, spacedim>> mapping; |
| 478 |
|
| 479 |
/** |
| 480 |
* (One-to-one) relation of points and cells. |
| 481 |
*/ |
| 482 |
bool unique_mapping; |
| 483 |
|
| 484 |
/** |
| 485 |
* Cache if all points passed in during reinit() have been found. |
| 486 |
*/ |
| 487 |
bool all_points_found_flag; |
| 488 |
|
| 489 |
/** |
| 490 |
* Since for each point multiple or no results can be available, the |
| 491 |
* pointers in this vector indicate the first and last entry associated |
| 492 |
* with a point in a CRS-like fashion. |
| 493 |
*/ |
| 494 |
std::vector<unsigned int> point_ptrs; |
| 495 |
|
| 496 |
/** |
| 497 |
* Permutation index within a recv buffer. |
| 498 |
*/ |
| 499 |
std::vector<unsigned int> recv_permutation; |
| 500 |
|
| 501 |
/** |
| 502 |
* Inverse of permutation index within a recv buffer. |
| 503 |
*/ |
| 504 |
std::vector<unsigned int> recv_permutation_inv; |
| 505 |
|
| 506 |
/** |
| 507 |
* Pointers of ranges within a receive buffer that are filled by ranks |
| 508 |
* specified by recv_ranks. |
| 509 |
*/ |
| 510 |
std::vector<unsigned int> recv_ptrs; |
| 511 |
|
| 512 |
/** |
| 513 |
* Ranks from where data is received. |
| 514 |
*/ |
| 515 |
std::vector<unsigned int> recv_ranks; |
| 516 |
|
| 517 |
/** |
| 518 |
* Point data sorted according to cells so that evaluation (incl. reading |
| 519 |
* of degrees of freedoms) needs to be performed only once per cell. |
| 520 |
*/ |
| 521 |
std::unique_ptr<CellData> cell_data; |
| 522 |
|
| 523 |
/** |
| 524 |
* Permutation index within a send buffer. |
| 525 |
*/ |
| 526 |
std::vector<unsigned int> send_permutation; |
| 527 |
|
| 528 |
/** |
| 529 |
* Inverse of permutation index within a send buffer. |
| 530 |
*/ |
| 531 |
std::vector<unsigned int> send_permutation_inv; |
| 532 |
|
| 533 |
/** |
| 534 |
* Ranks to send to. |
| 535 |
*/ |
| 536 |
std::vector<unsigned int> send_ranks; |
| 537 |
|
| 538 |
/** |
| 539 |
* Pointers of ranges within a send buffer to be sent to the ranks |
| 540 |
* specified by send_ranks. |
| 541 |
*/ |
| 542 |
std::vector<unsigned int> send_ptrs; |
| 543 |
|
| 544 |
/** |
| 545 |
* Buffer size (if internal sorting is requested) determined as max of: |
| 546 |
* |
| 547 |
* needed during evaluate_and_process() |
| 548 |
* - point data (sorted according to cells) -> input from cell loop |
| 549 |
* - point data (sorted according to ranks) -> to be sent |
| 550 |
* - memory for receiving data (by one process at a time) |
| 551 |
* |
| 552 |
* needed during process_and_evaluate() |
| 553 |
* - point data (sorted according to cells) -> output to cell loop |
| 554 |
* - point data (sorted according to ranks) -> to be sent |
| 555 |
* - memory for receiving data (by one process at a time) |
| 556 |
*/ |
| 557 |
unsigned int buffer_size_with_sorting; |
| 558 |
|
| 559 |
/** |
| 560 |
* Buffer size (if sorting is not requested); corresponds to the |
| 561 |
* number of evaluation points. |
| 562 |
*/ |
| 563 |
unsigned int buffer_size_without_sorting; |
| 564 |
}; |
| 565 |
|
| 566 |
namespace internal |
| 567 |
{ |
| 568 |
#ifdef DEAL_II_WITH_MPI |
| 569 |
/** |
| 570 |
* Pack @p data and send it via MPI_Isend. |
| 571 |
*/ |
| 572 |
template <typename T> |
| 573 |
std::enable_if_t<Utilities::MPI::is_mpi_type<T> == false, void> |
| 574 |
pack_and_isend(const ArrayView<const T> &data, |
| 575 |
const unsigned int rank, |
| 576 |
const unsigned int tag, |
| 577 |
const MPI_Comm comm, |
| 578 |
std::vector<std::vector<char>> &buffers, |
| 579 |
std::vector<MPI_Request> &requests) |
| 580 |
{ |
| 581 |
requests.emplace_back(MPI_Request()); |
| 582 |
|
| 583 |
buffers.emplace_back(Utilities::pack( |
| 584 |
std::vector<T>(data.data(), data.data() + data.size()), false)); |
| 585 |
|
| 586 |
const int ierr = MPI_Isend(buffers.back().data(), |
| 587 |
buffers.back().size(), |
| 588 |
MPI_CHAR, |
| 589 |
rank, |
| 590 |
tag, |
| 591 |
comm, |
| 592 |
&requests.back()); |
| 593 |
AssertThrowMPI(ierr); |
| 594 |
} |
| 595 |
|
| 596 |
|
| 597 |
|
| 598 |
/** |
| 599 |
* Above function specialized for data types supported by MPI |
| 600 |
* so that one can skip packing. |
| 601 |
*/ |
| 602 |
template <typename T> |
| 603 |
std::enable_if_t<Utilities::MPI::is_mpi_type<T> == true, void> |
| 604 |
pack_and_isend(const ArrayView<const T> &data, |
| 605 |
const unsigned int rank, |
| 606 |
const unsigned int tag, |
| 607 |
const MPI_Comm comm, |
| 608 |
std::vector<std::vector<char>> & /*buffers*/, |
| 609 |
std::vector<MPI_Request> &requests) |
| 610 |
{ |
| 611 |
requests.emplace_back(MPI_Request()); |
| 612 |
|
| 613 |
const int ierr = MPI_Isend(data.data(), |
| 614 |
data.size(), |
| 615 |
Utilities::MPI::mpi_type_id_for_type<T>, |
| 616 |
rank, |
| 617 |
tag, |
| 618 |
comm, |
| 619 |
&requests.back()); |
| 620 |
AssertThrowMPI(ierr); |
| 621 |
} |
| 622 |
|
| 623 |
|
| 624 |
|
| 625 |
/** |
| 626 |
* Above function specialized for Tenors objects. The underlying data type |
| 627 |
* might be supported by MPI so that one can skip packing. |
| 628 |
*/ |
| 629 |
template <int rank_, int dim, typename T> |
| 630 |
std::enable_if_t<Utilities::MPI::is_mpi_type<T> == true, void> |
| 631 |
pack_and_isend(const ArrayView<const Tensor<rank_, dim, T>> &data, |
| 632 |
const unsigned int rank, |
| 633 |
const unsigned int tag, |
| 634 |
const MPI_Comm comm, |
| 635 |
std::vector<std::vector<char>> &buffers, |
| 636 |
std::vector<MPI_Request> &requests) |
| 637 |
{ |
| 638 |
ArrayView<const T> data_(reinterpret_cast<const T *>(data.data()), |
| 639 |
data.size() * Utilities::pow(dim, rank_)); |
| 640 |
|
| 641 |
pack_and_isend(data_, rank, tag, comm, buffers, requests); |
| 642 |
} |
| 643 |
|
| 644 |
|
| 645 |
/** |
| 646 |
* Receive message, unpack it, and store the result in @p data. |
| 647 |
*/ |
| 648 |
template <typename T> |
| 649 |
std::enable_if_t<Utilities::MPI::is_mpi_type<T> == false, void> |
| 650 |
recv_and_unpack(const ArrayView<T> &data, |
| 651 |
const MPI_Comm comm, |
| 652 |
const MPI_Status &status, |
| 653 |
std::vector<char> &buffer) |
| 654 |
{ |
| 655 |
int message_length; |
| 656 |
int ierr = MPI_Get_count(&status, MPI_CHAR, &message_length); |
| 657 |
AssertThrowMPI(ierr); |
| 658 |
|
| 659 |
buffer.resize(message_length); |
| 660 |
|
| 661 |
ierr = MPI_Recv(buffer.data(), |
| 662 |
buffer.size(), |
| 663 |
MPI_CHAR, |
| 664 |
status.MPI_SOURCE, |
| 665 |
internal::Tags::remote_point_evaluation, |
| 666 |
comm, |
| 667 |
MPI_STATUS_IGNORE); |
| 668 |
AssertThrowMPI(ierr); |
| 669 |
|
| 670 |
// unpack data |
| 671 |
const auto temp = Utilities::unpack<std::vector<T>>(buffer, false); |
| 672 |
|
| 673 |
for (unsigned int i = 0; i < data.size(); ++i) |
| 674 |
data[i] = temp[i]; |
| 675 |
} |
| 676 |
|
| 677 |
|
| 678 |
|
| 679 |
/** |
| 680 |
* Above function specialized for data types supported by MPI |
| 681 |
* so that one can skip unpacking. |
| 682 |
*/ |
| 683 |
template <typename T> |
| 684 |
std::enable_if_t<Utilities::MPI::is_mpi_type<T> == true, void> |
| 685 |
recv_and_unpack(const ArrayView<T> &data, |
| 686 |
const MPI_Comm comm, |
| 687 |
const MPI_Status &status, |
| 688 |
std::vector<char> & /*buffer*/) |
| 689 |
{ |
| 690 |
const auto ierr = MPI_Recv(data.data(), |
| 691 |
data.size(), |
| 692 |
Utilities::MPI::mpi_type_id_for_type<T>, |
| 693 |
status.MPI_SOURCE, |
| 694 |
internal::Tags::remote_point_evaluation, |
| 695 |
comm, |
| 696 |
MPI_STATUS_IGNORE); |
| 697 |
AssertThrowMPI(ierr); |
| 698 |
} |
| 699 |
|
| 700 |
|
| 701 |
|
| 702 |
/** |
| 703 |
* Above function specialized for Tensor objects. The underlying data type |
| 704 |
* might be supported by MPI so that one can skip unpacking. |
| 705 |
*/ |
| 706 |
template <int rank_, int dim, typename T> |
| 707 |
std::enable_if_t<Utilities::MPI::is_mpi_type<T> == true, void> |
| 708 |
recv_and_unpack(const ArrayView<Tensor<rank_, dim, T>> &data, |
| 709 |
const MPI_Comm comm, |
| 710 |
const MPI_Status &status, |
| 711 |
std::vector<char> &buffer) |
| 712 |
{ |
| 713 |
const ArrayView<T> data_(reinterpret_cast<T *>(data.data()), |
| 714 |
data.size() * Utilities::pow(dim, rank_)); |
| 715 |
|
| 716 |
recv_and_unpack(data_, comm, status, buffer); |
| 717 |
} |
| 718 |
#endif |
| 719 |
} // namespace internal |
| 720 |
|
| 721 |
|
| 722 |
|
| 723 |
template <int dim, int spacedim> |
| 724 |
template <typename DataType> |
| 725 |
ArrayView<DataType> |
| 726 |
RemotePointEvaluation<dim, spacedim>::CellData::get_data_view( |
| 727 |
const unsigned int cell, |
| 728 |
const ArrayView<DataType> &values) const |
| 729 |
{ |
| 730 |
AssertIndexRange(cell, cells.size()); |
| 731 |
return {values.data() + reference_point_ptrs[cell], |
| 732 |
reference_point_ptrs[cell + 1] - reference_point_ptrs[cell]}; |
| 733 |
} |
| 734 |
|
| 735 |
|
| 736 |
|
| 737 |
template <int dim, int spacedim> |
| 738 |
template <typename DataType, unsigned int n_components> |
| 739 |
void |
| 740 |
RemotePointEvaluation<dim, spacedim>::evaluate_and_process( |
| 741 |
std::vector<DataType> &output, |
| 742 |
std::vector<DataType> &buffer, |
| 743 |
const std::function<void(const ArrayView<DataType> &, const CellData &)> |
| 744 |
&evaluation_function, |
| 745 |
const bool sort_data) const |
| 746 |
{ |
| 747 |
#ifndef DEAL_II_WITH_MPI |
| 748 |
Assert(false, ExcNeedsMPI()); |
| 749 |
(void)output; |
| 750 |
(void)buffer; |
| 751 |
(void)evaluation_function; |
| 752 |
(void)sort_data; |
| 753 |
#else |
| 754 |
static CollectiveMutex mutex; |
| 755 |
CollectiveMutex::ScopedLock lock(mutex, tria->get_mpi_communicator()); |
| 756 |
|
| 757 |
const unsigned int my_rank = |
| 758 |
Utilities::MPI::this_mpi_process(tria->get_mpi_communicator()); |
| 759 |
|
| 760 |
// allocate memory for output and buffer |
| 761 |
output.resize(point_ptrs.back() * n_components); |
| 762 |
|
| 763 |
buffer.resize( |
| 764 |
(sort_data ? buffer_size_with_sorting : buffer_size_without_sorting) * |
| 765 |
n_components); |
| 766 |
|
| 767 |
// ... for evaluation |
| 768 |
ArrayView<DataType> buffer_eval(buffer.data(), |
| 769 |
send_permutation.size() * n_components); |
| 770 |
|
| 771 |
// ... for communication (send) |
| 772 |
ArrayView<DataType> buffer_send( |
| 773 |
sort_data ? (buffer.data() + send_permutation.size() * n_components) : |
| 774 |
buffer.data(), |
| 775 |
send_permutation.size() * n_components); |
| 776 |
|
| 777 |
// more arrays |
| 778 |
std::vector<MPI_Request> send_requests; |
| 779 |
std::vector<std::vector<char>> send_buffers_packed; |
| 780 |
std::vector<char> recv_buffer_packed; |
| 781 |
|
| 782 |
// evaluate functions at points |
| 783 |
evaluation_function(buffer_eval, *cell_data); |
| 784 |
|
| 785 |
// sort for communication (optional) |
| 786 |
if (sort_data) |
| 787 |
{ |
| 788 |
const auto my_rank_local_recv_ptr = |
| 789 |
std::find(recv_ranks.begin(), recv_ranks.end(), my_rank); |
| 790 |
|
| 791 |
if (my_rank_local_recv_ptr != recv_ranks.end()) |
| 792 |
{ |
| 793 |
const unsigned int my_rank_local_recv = |
| 794 |
std::distance(recv_ranks.begin(), my_rank_local_recv_ptr); |
| 795 |
const unsigned int my_rank_local_send = std::distance( |
| 796 |
send_ranks.begin(), |
| 797 |
std::find(send_ranks.begin(), send_ranks.end(), my_rank)); |
| 798 |
const unsigned int start = send_ptrs[my_rank_local_send]; |
| 799 |
const unsigned int end = send_ptrs[my_rank_local_send + 1]; |
| 800 |
const unsigned int *recv_ptr = |
| 801 |
recv_permutation.data() + recv_ptrs[my_rank_local_recv]; |
| 802 |
for (unsigned int i = 0; i < send_permutation.size(); ++i) |
| 803 |
{ |
| 804 |
const unsigned int send_index = send_permutation[i]; |
| 805 |
|
| 806 |
if (start <= send_index && send_index < end) |
| 807 |
// local data -> can be copied to output directly |
| 808 |
for (unsigned int c = 0; c < n_components; ++c) |
| 809 |
output[recv_ptr[send_index - start] * n_components + c] = |
| 810 |
buffer_eval[i * n_components + c]; |
| 811 |
else |
| 812 |
// data to be sent |
| 813 |
for (unsigned int c = 0; c < n_components; ++c) |
| 814 |
buffer_send[send_index * n_components + c] = |
| 815 |
buffer_eval[i * n_components + c]; |
| 816 |
} |
| 817 |
} |
| 818 |
else |
| 819 |
{ |
| 820 |
for (unsigned int i = 0; i < send_permutation.size(); ++i) |
| 821 |
for (unsigned int c = 0; c < n_components; ++c) |
| 822 |
buffer_send[send_permutation[i] * n_components + c] = |
| 823 |
buffer_eval[i * n_components + c]; |
| 824 |
} |
| 825 |
} |
| 826 |
|
| 827 |
// send data |
| 828 |
send_buffers_packed.reserve(send_ranks.size()); |
| 829 |
send_requests.reserve(send_ranks.size()); |
| 830 |
|
| 831 |
for (unsigned int i = 0; i < send_ranks.size(); ++i) |
| 832 |
{ |
| 833 |
if (send_ranks[i] == my_rank) |
| 834 |
continue; |
| 835 |
|
| 836 |
internal::pack_and_isend( |
| 837 |
ArrayView<const DataType>( |
| 838 |
buffer_send.begin() + send_ptrs[i] * n_components, |
| 839 |
(send_ptrs[i + 1] - send_ptrs[i]) * n_components), |
| 840 |
send_ranks[i], |
| 841 |
internal::Tags::remote_point_evaluation, |
| 842 |
tria->get_mpi_communicator(), |
| 843 |
send_buffers_packed, |
| 844 |
send_requests); |
| 845 |
} |
| 846 |
|
| 847 |
// copy local data directly to output if no sorting is requested |
| 848 |
if (!sort_data) |
| 849 |
{ |
| 850 |
const auto my_rank_local_recv_ptr = |
| 851 |
std::find(recv_ranks.begin(), recv_ranks.end(), my_rank); |
| 852 |
|
| 853 |
if (my_rank_local_recv_ptr != recv_ranks.end()) |
| 854 |
{ |
| 855 |
const unsigned int my_rank_local_recv = |
| 856 |
std::distance(recv_ranks.begin(), my_rank_local_recv_ptr); |
| 857 |
const unsigned int my_rank_local_send = std::distance( |
| 858 |
send_ranks.begin(), |
| 859 |
std::find(send_ranks.begin(), send_ranks.end(), my_rank)); |
| 860 |
|
| 861 |
for (unsigned int j = recv_ptrs[my_rank_local_recv], |
| 862 |
k = send_ptrs[my_rank_local_send]; |
| 863 |
j < recv_ptrs[my_rank_local_recv + 1]; |
| 864 |
++j, ++k) |
| 865 |
for (unsigned int c = 0; c < n_components; ++c) |
| 866 |
output[j * n_components + c] = |
| 867 |
buffer_eval[k * n_components + c]; |
| 868 |
} |
| 869 |
} |
| 870 |
|
| 871 |
// receive data |
| 872 |
for (unsigned int i = 0; i < recv_ranks.size(); ++i) |
| 873 |
{ |
| 874 |
if (recv_ranks[i] == my_rank) |
| 875 |
continue; |
| 876 |
|
| 877 |
// receive remote data |
| 878 |
MPI_Status status; |
| 879 |
int ierr = MPI_Probe(MPI_ANY_SOURCE, |
| 880 |
internal::Tags::remote_point_evaluation, |
| 881 |
tria->get_mpi_communicator(), |
| 882 |
&status); |
| 883 |
AssertThrowMPI(ierr); |
| 884 |
|
| 885 |
const auto ptr = |
| 886 |
std::find(recv_ranks.begin(), recv_ranks.end(), status.MPI_SOURCE); |
| 887 |
|
| 888 |
Assert(ptr != recv_ranks.end(), ExcNotImplemented()); |
| 889 |
|
| 890 |
const unsigned int j = std::distance(recv_ranks.begin(), ptr); |
| 891 |
|
| 892 |
// ... for communication (recv) |
| 893 |
ArrayView<DataType> recv_buffer( |
| 894 |
sort_data ? |
| 895 |
(buffer.data() + send_permutation.size() * 2 * n_components) : |
| 896 |
(output.data() + recv_ptrs[j] * n_components), |
| 897 |
(recv_ptrs[j + 1] - recv_ptrs[j]) * n_components); |
| 898 |
|
| 899 |
internal::recv_and_unpack(recv_buffer, |
| 900 |
tria->get_mpi_communicator(), |
| 901 |
status, |
| 902 |
recv_buffer_packed); |
| 903 |
|
| 904 |
if (sort_data) |
| 905 |
{ |
| 906 |
// sort data into output vector (optional) |
| 907 |
for (unsigned int i = recv_ptrs[j], k = 0; i < recv_ptrs[j + 1]; |
| 908 |
++i, ++k) |
| 909 |
for (unsigned int c = 0; c < n_components; ++c) |
| 910 |
output[recv_permutation[i] * n_components + c] = |
| 911 |
recv_buffer[k * n_components + c]; |
| 912 |
} |
| 913 |
} |
| 914 |
|
| 915 |
// make sure all messages have been sent |
| 916 |
if (!send_requests.empty()) |
| 917 |
{ |
| 918 |
const int ierr = MPI_Waitall(send_requests.size(), |
| 919 |
send_requests.data(), |
| 920 |
MPI_STATUSES_IGNORE); |
| 921 |
AssertThrowMPI(ierr); |
| 922 |
} |
| 923 |
#endif |
| 924 |
} |
| 925 |
|
| 926 |
|
| 927 |
template <int dim, int spacedim> |
| 928 |
template <typename DataType, unsigned int n_components> |
| 929 |
std::vector<DataType> |
| 930 |
RemotePointEvaluation<dim, spacedim>::evaluate_and_process( |
| 931 |
const std::function<void(const ArrayView<DataType> &, const CellData &)> |
| 932 |
&evaluation_function, |
| 933 |
const bool sort_data) const |
| 934 |
{ |
| 935 |
std::vector<DataType> output; |
| 936 |
std::vector<DataType> buffer; |
| 937 |
|
| 938 |
this->evaluate_and_process<DataType, n_components>(output, |
| 939 |
buffer, |
| 940 |
evaluation_function, |
| 941 |
sort_data); |
| 942 |
|
| 943 |
return output; |
| 944 |
} |
| 945 |
|
| 946 |
|
| 947 |
|
| 948 |
template <int dim, int spacedim> |
| 949 |
template <typename DataType, unsigned int n_components> |
| 950 |
void |
| 951 |
RemotePointEvaluation<dim, spacedim>::process_and_evaluate( |
| 952 |
const std::vector<DataType> &input, |
| 953 |
std::vector<DataType> &buffer, |
| 954 |
const std::function<void(const ArrayView<const DataType> &, |
| 955 |
const CellData &)> &evaluation_function, |
| 956 |
const bool sort_data) const |
| 957 |
{ |
| 958 |
#ifndef DEAL_II_WITH_MPI |
| 959 |
Assert(false, ExcNeedsMPI()); |
| 960 |
(void)input; |
| 961 |
(void)buffer; |
| 962 |
(void)evaluation_function; |
| 963 |
(void)sort_data; |
| 964 |
#else |
| 965 |
static CollectiveMutex mutex; |
| 966 |
CollectiveMutex::ScopedLock lock(mutex, tria->get_mpi_communicator()); |
| 967 |
|
| 968 |
const unsigned int my_rank = |
| 969 |
Utilities::MPI::this_mpi_process(tria->get_mpi_communicator()); |
| 970 |
|
| 971 |
// allocate memory for buffer |
| 972 |
const auto &point_ptrs = this->get_point_ptrs(); |
| 973 |
|
| 974 |
AssertDimension(input.size(), |
| 975 |
sort_data ? ((point_ptrs.size() - 1) * n_components) : |
| 976 |
(point_ptrs.back() * n_components)); |
| 977 |
buffer.resize( |
| 978 |
(sort_data ? buffer_size_with_sorting : buffer_size_without_sorting) * |
| 979 |
n_components); |
| 980 |
|
| 981 |
// ... for evaluation |
| 982 |
ArrayView<DataType> buffer_eval(buffer.data(), |
| 983 |
send_permutation.size() * n_components); |
| 984 |
|
| 985 |
// ... for communication (send) |
| 986 |
ArrayView<DataType> buffer_send( |
| 987 |
sort_data ? (buffer.data() + send_permutation.size() * n_components) : |
| 988 |
const_cast<DataType *>(input.data()), |
| 989 |
point_ptrs.back() * n_components); |
| 990 |
|
| 991 |
// more arrays |
| 992 |
std::vector<MPI_Request> send_requests; |
| 993 |
std::vector<std::vector<char>> send_buffers_packed; |
| 994 |
std::vector<char> recv_buffer_packed; |
| 995 |
|
| 996 |
// sort for communication (and duplicate data if necessary) |
| 997 |
|
| 998 |
if (sort_data) |
| 999 |
{ |
| 1000 |
const auto my_rank_local_recv_ptr = |
| 1001 |
std::find(recv_ranks.begin(), recv_ranks.end(), my_rank); |
| 1002 |
|
| 1003 |
if (my_rank_local_recv_ptr != recv_ranks.end()) |
| 1004 |
{ |
| 1005 |
// optimize the case if we have our own rank among the possible |
| 1006 |
// list |
| 1007 |
const unsigned int my_rank_local_recv = |
| 1008 |
std::distance(recv_ranks.begin(), my_rank_local_recv_ptr); |
| 1009 |
const unsigned int my_rank_local_send = std::distance( |
| 1010 |
send_ranks.begin(), |
| 1011 |
std::find(send_ranks.begin(), send_ranks.end(), my_rank)); |
| 1012 |
|
| 1013 |
const unsigned int start = recv_ptrs[my_rank_local_recv]; |
| 1014 |
const unsigned int end = recv_ptrs[my_rank_local_recv + 1]; |
| 1015 |
const unsigned int *send_ptr = |
| 1016 |
send_permutation_inv.data() + send_ptrs[my_rank_local_send]; |
| 1017 |
for (unsigned int i = 0, k = 0; i < point_ptrs.size() - 1; ++i) |
| 1018 |
{ |
| 1019 |
const unsigned int next = point_ptrs[i + 1]; |
| 1020 |
for (unsigned int j = point_ptrs[i]; j < next; ++j, ++k) |
| 1021 |
{ |
| 1022 |
const unsigned int recv_index = recv_permutation_inv[k]; |
| 1023 |
|
| 1024 |
// local data -> can be copied to final buffer directly |
| 1025 |
if (start <= recv_index && recv_index < end) |
| 1026 |
for (unsigned int c = 0; c < n_components; ++c) |
| 1027 |
buffer_eval[send_ptr[recv_index - start] * |
| 1028 |
n_components + |
| 1029 |
c] = input[i * n_components + c]; |
| 1030 |
else // data to be sent |
| 1031 |
for (unsigned int c = 0; c < n_components; ++c) |
| 1032 |
buffer_send[recv_index * n_components + c] = |
| 1033 |
input[i * n_components + c]; |
| 1034 |
} |
| 1035 |
} |
| 1036 |
} |
| 1037 |
else |
| 1038 |
{ |
| 1039 |
for (unsigned int i = 0, k = 0; i < point_ptrs.size() - 1; ++i) |
| 1040 |
for (unsigned int j = point_ptrs[i]; j < point_ptrs[i + 1]; |
| 1041 |
++j, ++k) |
| 1042 |
for (unsigned int c = 0; c < n_components; ++c) |
| 1043 |
buffer_send[recv_permutation_inv[k] * n_components + c] = |
| 1044 |
input[i * n_components + c]; |
| 1045 |
} |
| 1046 |
} |
| 1047 |
|
| 1048 |
// send data |
| 1049 |
send_buffers_packed.reserve(recv_ranks.size()); |
| 1050 |
send_requests.reserve(recv_ranks.size()); |
| 1051 |
|
| 1052 |
for (unsigned int i = 0; i < recv_ranks.size(); ++i) |
| 1053 |
{ |
| 1054 |
if (recv_ranks[i] == my_rank) |
| 1055 |
continue; |
| 1056 |
|
| 1057 |
internal::pack_and_isend( |
| 1058 |
ArrayView<const DataType>( |
| 1059 |
buffer_send.begin() + recv_ptrs[i] * n_components, |
| 1060 |
(recv_ptrs[i + 1] - recv_ptrs[i]) * n_components), |
| 1061 |
recv_ranks[i], |
| 1062 |
internal::Tags::remote_point_evaluation, |
| 1063 |
tria->get_mpi_communicator(), |
| 1064 |
send_buffers_packed, |
| 1065 |
send_requests); |
| 1066 |
} |
| 1067 |
|
| 1068 |
// copy local data directly to output if no sorting is requested |
| 1069 |
if (!sort_data) |
| 1070 |
{ |
| 1071 |
const auto my_rank_local_recv_ptr = |
| 1072 |
std::find(recv_ranks.begin(), recv_ranks.end(), my_rank); |
| 1073 |
|
| 1074 |
if (my_rank_local_recv_ptr != recv_ranks.end()) |
| 1075 |
{ |
| 1076 |
const unsigned int my_rank_local_recv = |
| 1077 |
std::distance(recv_ranks.begin(), my_rank_local_recv_ptr); |
| 1078 |
const unsigned int my_rank_local_send = std::distance( |
| 1079 |
send_ranks.begin(), |
| 1080 |
std::find(send_ranks.begin(), send_ranks.end(), my_rank)); |
| 1081 |
|
| 1082 |
for (unsigned int j = recv_ptrs[my_rank_local_recv], |
| 1083 |
k = send_ptrs[my_rank_local_send]; |
| 1084 |
j < recv_ptrs[my_rank_local_recv + 1]; |
| 1085 |
++j, ++k) |
| 1086 |
for (unsigned int c = 0; c < n_components; ++c) |
| 1087 |
buffer_eval[k * n_components + c] = |
| 1088 |
input[j * n_components + c]; |
| 1089 |
} |
| 1090 |
} |
| 1091 |
|
| 1092 |
for (unsigned int i = 0; i < send_ranks.size(); ++i) |
| 1093 |
{ |
| 1094 |
if (send_ranks[i] == my_rank) |
| 1095 |
continue; |
| 1096 |
|
| 1097 |
// receive remote data |
| 1098 |
MPI_Status status; |
| 1099 |
int ierr = MPI_Probe(MPI_ANY_SOURCE, |
| 1100 |
internal::Tags::remote_point_evaluation, |
| 1101 |
tria->get_mpi_communicator(), |
| 1102 |
&status); |
| 1103 |
AssertThrowMPI(ierr); |
| 1104 |
|
| 1105 |
// write data into buffer vector |
| 1106 |
const auto ptr = |
| 1107 |
std::find(send_ranks.begin(), send_ranks.end(), status.MPI_SOURCE); |
| 1108 |
|
| 1109 |
Assert(ptr != send_ranks.end(), ExcNotImplemented()); |
| 1110 |
|
| 1111 |
const unsigned int j = std::distance(send_ranks.begin(), ptr); |
| 1112 |
|
| 1113 |
ArrayView<DataType> recv_buffer( |
| 1114 |
sort_data ? |
| 1115 |
(buffer.data() + |
| 1116 |
(point_ptrs.back() + send_permutation.size()) * n_components) : |
| 1117 |
(buffer.data() + send_ptrs[j] * n_components), |
| 1118 |
(send_ptrs[j + 1] - send_ptrs[j]) * n_components); |
| 1119 |
|
| 1120 |
internal::recv_and_unpack(recv_buffer, |
| 1121 |
tria->get_mpi_communicator(), |
| 1122 |
status, |
| 1123 |
recv_buffer_packed); |
| 1124 |
|
| 1125 |
if (sort_data) |
| 1126 |
{ |
| 1127 |
for (unsigned int i = send_ptrs[j], k = 0; i < send_ptrs[j + 1]; |
| 1128 |
++i, ++k) |
| 1129 |
for (unsigned int c = 0; c < n_components; ++c) |
| 1130 |
buffer_eval[send_permutation_inv[i] * n_components + c] = |
| 1131 |
recv_buffer[k * n_components + c]; |
| 1132 |
} |
| 1133 |
} |
| 1134 |
|
| 1135 |
if (!send_requests.empty()) |
| 1136 |
{ |
| 1137 |
const int ierr = MPI_Waitall(send_requests.size(), |
| 1138 |
send_requests.data(), |
| 1139 |
MPI_STATUSES_IGNORE); |
| 1140 |
AssertThrowMPI(ierr); |
| 1141 |
} |
| 1142 |
|
| 1143 |
// evaluate function at points |
| 1144 |
evaluation_function(buffer_eval, *cell_data); |
| 1145 |
#endif |
| 1146 |
} |
| 1147 |
|
| 1148 |
|
| 1149 |
|
| 1150 |
template <int dim, int spacedim> |
| 1151 |
template <typename DataType, unsigned int n_components> |
| 1152 |
void |
| 1153 |
RemotePointEvaluation<dim, spacedim>::process_and_evaluate( |
| 1154 |
const std::vector<DataType> &input, |
| 1155 |
const std::function<void(const ArrayView<const DataType> &, |
| 1156 |
const CellData &)> &evaluation_function, |
| 1157 |
const bool sort_data) const |
| 1158 |
{ |
| 1159 |
std::vector<DataType> buffer; |
| 1160 |
this->process_and_evaluate<DataType, n_components>(input, |
| 1161 |
buffer, |
| 1162 |
evaluation_function, |
| 1163 |
sort_data); |
| 1164 |
} |
| 1165 |
|
| 1166 |
} // end of namespace MPI |
| 1167 |
} // end of namespace Utilities |
| 1168 |
|
| 1169 |
|
| 1170 |
DEAL_II_NAMESPACE_CLOSE |
| 1171 |
|
| 1172 |
#endif |