Debian Codegraph @

include/deal.II/base/partitioner.templates.h

raw
1 // ------------------------------------------------------------------------
2 //
3 // SPDX-License-Identifier: LGPL-2.1-or-later
4 // Copyright (C) 2017 - 2025 by the deal.II authors
5 //
6 // This file is part of the deal.II library.
7 //
8 // Part of the source code is dual licensed under Apache-2.0 WITH
9 // LLVM-exception OR LGPL-2.1-or-later. Detailed license information
10 // governing the source code and code contributions can be found in
11 // LICENSE.md and CONTRIBUTING.md at the top level directory of deal.II.
12 //
13 // ------------------------------------------------------------------------
14
15 #ifndef dealii_partitioner_templates_h
16 #define dealii_partitioner_templates_h
17
18 #include <deal.II/base/config.h>
19
20 #include <deal.II/base/mpi_tags.h>
21 #include <deal.II/base/partitioner.h>
22
23 #include <deal.II/lac/la_parallel_vector.h>
24
25 #include <limits>
26 #include <type_traits>
27
28
29 DEAL_II_NAMESPACE_OPEN
30
31 namespace Utilities
32 {
33 namespace MPI
34 {
35 #ifndef DOXYGEN
36
37 # ifdef DEAL_II_WITH_MPI
38
39 template <typename Number, typename MemorySpaceType>
40 void
41 Partitioner::export_to_ghosted_array_start(
42 const unsigned int communication_channel,
43 const ArrayView<const Number, MemorySpaceType> &locally_owned_array,
44 const ArrayView<Number, MemorySpaceType> &temporary_storage,
45 const ArrayView<Number, MemorySpaceType> &ghost_array,
46 std::vector<MPI_Request> &requests) const
47 {
48 AssertDimension(temporary_storage.size(), n_import_indices());
49 AssertIndexRange(communication_channel, 200);
50 Assert(ghost_array.size() == n_ghost_indices() ||
51 ghost_array.size() == n_ghost_indices_in_larger_set,
52 ExcGhostIndexArrayHasWrongSize(ghost_array.size(),
53 n_ghost_indices(),
54 n_ghost_indices_in_larger_set));
55
56 const unsigned int n_import_targets = import_targets_data.size();
57 const unsigned int n_ghost_targets = ghost_targets_data.size();
58
59 if (n_import_targets > 0)
60 AssertDimension(locally_owned_array.size(), locally_owned_size());
61
62 Assert(requests.empty(),
63 ExcMessage("Another operation seems to still be running. "
64 "Call update_ghost_values_finish() first."));
65
66 const unsigned int mpi_tag =
67 Utilities::MPI::internal::Tags::partitioner_export_start +
68 communication_channel;
69 Assert(mpi_tag <= Utilities::MPI::internal::Tags::partitioner_export_end,
70 ExcInternalError());
71
72 // Need to send and receive the data. Use non-blocking communication,
73 // where it is usually less overhead to first initiate the receive and
74 // then actually send the data
75 requests.resize(n_import_targets + n_ghost_targets);
76
77 // as a ghost array pointer, put the data at the end of the given ghost
78 // array in case we want to fill only a subset of the ghosts so that we
79 // can move data to the right position in a forward loop in the _finish
80 // function.
81 AssertIndexRange(n_ghost_indices(), n_ghost_indices_in_larger_set + 1);
82 const bool use_larger_set =
83 (n_ghost_indices_in_larger_set > n_ghost_indices() &&
84 ghost_array.size() == n_ghost_indices_in_larger_set);
85 Number *ghost_array_ptr =
86 use_larger_set ? ghost_array.data() + n_ghost_indices_in_larger_set -
87 n_ghost_indices() :
88 ghost_array.data();
89
90 for (unsigned int i = 0; i < n_ghost_targets; ++i)
91 {
92 // allow writing into ghost indices even though we are in a
93 // const function
94 const int ierr =
95 MPI_Irecv(ghost_array_ptr,
96 ghost_targets_data[i].second * sizeof(Number),
97 MPI_BYTE,
98 ghost_targets_data[i].first,
99 mpi_tag,
100 communicator,
101 &requests[i]);
102 AssertThrowMPI(ierr);
103 ghost_array_ptr += ghost_targets_data[i].second;
104 }
105
106 Number *temp_array_ptr = temporary_storage.data();
107 # if defined(DEAL_II_MPI_WITH_DEVICE_SUPPORT)
108 // When using device-aware MPI, the set of local indices that are ghosts
109 // indices on other processors is expanded in arrays. This is for
110 // performance reasons as this can significantly decrease the number of
111 // kernel launched. The indices are expanded the first time the function
112 // is called.
113 if ((std::is_same_v<MemorySpaceType, MemorySpace::Default>)&&(
114 import_indices_plain_dev.empty()))
115 initialize_import_indices_plain_dev();
116 # endif
117
118 for (unsigned int i = 0; i < n_import_targets; ++i)
119 {
120 # if defined(DEAL_II_MPI_WITH_DEVICE_SUPPORT)
121 if constexpr (std::is_same_v<MemorySpaceType, MemorySpace::Default>)
122 {
123 const auto chunk_size = import_indices_plain_dev[i].size();
124 using IndexType = decltype(chunk_size);
125
126 auto import_indices = import_indices_plain_dev[i];
127 auto locally_owned_array_data = locally_owned_array.data();
128 MemorySpace::Default::kokkos_space::execution_space exec;
129 Kokkos::parallel_for(
130 "dealii::fill temp_array_ptr",
131 Kokkos::RangePolicy<
132 MemorySpace::Default::kokkos_space::execution_space>(
133 exec, 0, chunk_size),
134 KOKKOS_LAMBDA(IndexType idx) {
135 temp_array_ptr[idx] =
136 locally_owned_array_data[import_indices[idx]];
137 });
138 exec.fence();
139 }
140 else
141 # endif
142 {
143 // copy the data to be sent to the import_data field
144 std::vector<std::pair<unsigned int, unsigned int>>::const_iterator
145 my_imports = import_indices_data.begin() +
146 import_indices_chunks_by_rank_data[i],
147 end_my_imports = import_indices_data.begin() +
148 import_indices_chunks_by_rank_data[i + 1];
149 unsigned int index = 0;
150 for (; my_imports != end_my_imports; ++my_imports)
151 {
152 const unsigned int chunk_size =
153 my_imports->second - my_imports->first;
154 {
155 std::memcpy(temp_array_ptr + index,
156 locally_owned_array.data() + my_imports->first,
157 chunk_size * sizeof(Number));
158 }
159 index += chunk_size;
160 }
161
162 AssertDimension(index, import_targets_data[i].second);
163 }
164
165 // start the send operations
166 const int ierr =
167 MPI_Isend(temp_array_ptr,
168 import_targets_data[i].second * sizeof(Number),
169 MPI_BYTE,
170 import_targets_data[i].first,
171 mpi_tag,
172 communicator,
173 &requests[n_ghost_targets + i]);
174 AssertThrowMPI(ierr);
175 temp_array_ptr += import_targets_data[i].second;
176 }
177 }
178
179
180
181 template <typename Number, typename MemorySpaceType>
182 void
183 Partitioner::export_to_ghosted_array_finish(
184 const ArrayView<Number, MemorySpaceType> &ghost_array,
185 std::vector<MPI_Request> &requests) const
186 {
187 Assert(ghost_array.size() == n_ghost_indices() ||
188 ghost_array.size() == n_ghost_indices_in_larger_set,
189 ExcGhostIndexArrayHasWrongSize(ghost_array.size(),
190 n_ghost_indices(),
191 n_ghost_indices_in_larger_set));
192
193 // wait for both sends and receives to complete, even though only
194 // receives are really necessary. this gives (much) better performance
195 AssertDimension(ghost_targets().size() + import_targets().size(),
196 requests.size());
197 if (requests.size() > 0)
198 {
199 const int ierr =
200 MPI_Waitall(requests.size(), requests.data(), MPI_STATUSES_IGNORE);
201 AssertThrowMPI(ierr);
202 }
203 requests.resize(0);
204
205 // in case we only sent a subset of indices, we now need to move the data
206 // to the correct positions and delete the old content
207 if (n_ghost_indices_in_larger_set > n_ghost_indices() &&
208 ghost_array.size() == n_ghost_indices_in_larger_set)
209 {
210 unsigned int offset =
211 n_ghost_indices_in_larger_set - n_ghost_indices();
212 // must copy ghost data into extended ghost array
213 for (const auto &ghost_range : ghost_indices_subset_data)
214 {
215 if (offset > ghost_range.first)
216 {
217 const unsigned int chunk_size =
218 ghost_range.second - ghost_range.first;
219 if constexpr (std::is_same_v<MemorySpaceType,
220 MemorySpace::Host>)
221 {
222 // If source and destination are overlapping, we must be
223 // careful to use an appropriate copy function.
224 if (ghost_range.first > offset)
225 std::copy_backward(ghost_array.data() + offset,
226 ghost_array.data() + offset +
227 chunk_size,
228 ghost_array.data() +
229 ghost_range.first);
230 else
231 std::copy(ghost_array.data() + offset,
232 ghost_array.data() + offset + chunk_size,
233 ghost_array.data() + ghost_range.first);
234 std::fill(ghost_array.data() +
235 std::max(ghost_range.second, offset),
236 ghost_array.data() + offset + chunk_size,
237 Number{});
238 }
239 else
240 {
241 Kokkos::View<const Number *,
242 MemorySpace::Default::kokkos_space>
243 ghost_src_view(ghost_array.data() + offset, chunk_size);
244 Kokkos::View<Number *, MemorySpace::Default::kokkos_space>
245 ghost_dst_view(ghost_array.data() + ghost_range.first,
246 chunk_size);
247
248 // If source and destination are overlapping, we can't
249 // just call deep_copy but must copy the data to a buffer
250 // first.
251 if ((offset < ghost_range.first &&
252 ghost_range.first < offset + chunk_size) ||
253 (ghost_range.first < offset &&
254 offset < ghost_range.first + chunk_size))
255 {
256 Kokkos::View<Number *,
257 MemorySpace::Default::kokkos_space>
258 copy(Kokkos::view_alloc(
259 "copy", Kokkos::WithoutInitializing),
260 chunk_size);
261 Kokkos::deep_copy(copy, ghost_src_view);
262 Kokkos::deep_copy(ghost_dst_view, copy);
263 }
264 else
265 {
266 Kokkos::deep_copy(ghost_dst_view, ghost_src_view);
267 }
268 Kokkos::deep_copy(
269 Kokkos::View<Number *,
270 MemorySpace::Default::kokkos_space>(
271 ghost_array.data() +
272 std::max(ghost_range.second, offset),
273 (offset + chunk_size -
274 std::max(ghost_range.second, offset))),
275 0);
276 }
277 offset += chunk_size;
278 }
279 else
280 {
281 AssertDimension(offset, ghost_range.first);
282 break;
283 }
284 }
285 }
286 }
287
288
289
290 template <typename Number, typename MemorySpaceType>
291 void
292 Partitioner::import_from_ghosted_array_start(
293 const VectorOperation::values vector_operation,
294 const unsigned int communication_channel,
295 const ArrayView<Number, MemorySpaceType> &ghost_array,
296 const ArrayView<Number, MemorySpaceType> &temporary_storage,
297 std::vector<MPI_Request> &requests) const
298 {
299 AssertDimension(temporary_storage.size(), n_import_indices());
300 AssertIndexRange(communication_channel, 200);
301 Assert(ghost_array.size() == n_ghost_indices() ||
302 ghost_array.size() == n_ghost_indices_in_larger_set,
303 ExcGhostIndexArrayHasWrongSize(ghost_array.size(),
304 n_ghost_indices(),
305 n_ghost_indices_in_larger_set));
306
307 (void)vector_operation;
308
309 // nothing to do for insert (only need to zero ghost entries in
310 // compress_finish()). in debug mode we want to check consistency of the
311 // inserted data, therefore the communication is still initialized.
312 // Having different code in debug and optimized mode is somewhat
313 // dangerous, but it really saves communication so do it anyway
314 if constexpr (running_in_debug_mode() == false)
315 if (vector_operation == VectorOperation::insert)
316 return;
317
318 // nothing to do when we neither have import
319 // nor ghost indices.
320 if (n_ghost_indices() == 0 && n_import_indices() == 0)
321 return;
322
323 const unsigned int n_import_targets = import_targets_data.size();
324 const unsigned int n_ghost_targets = ghost_targets_data.size();
325
326 Assert(requests.empty(),
327 ExcMessage("Another compress operation seems to still be running. "
328 "Call compress_finish() first."));
329
330 // Need to send and receive the data. Use non-blocking communication,
331 // where it is generally less overhead to first initiate the receive and
332 // then actually send the data
333
334 const unsigned int mpi_tag =
335 Utilities::MPI::internal::Tags::partitioner_import_start +
336 communication_channel;
337 Assert(mpi_tag <= Utilities::MPI::internal::Tags::partitioner_import_end,
338 ExcInternalError());
339 requests.resize(n_import_targets + n_ghost_targets);
340
341 // initiate the receive operations
342 Number *temp_array_ptr = temporary_storage.data();
343 for (unsigned int i = 0; i < n_import_targets; ++i)
344 {
345 AssertThrow(
346 static_cast<std::size_t>(import_targets_data[i].second) *
347 sizeof(Number) <
348 static_cast<std::size_t>(std::numeric_limits<int>::max()),
349 ExcMessage("Index overflow: Maximum message size in MPI is 2GB. "
350 "The number of ghost entries times the size of 'Number' "
351 "exceeds this value. This is not supported."));
352 const int ierr =
353 MPI_Irecv(temp_array_ptr,
354 import_targets_data[i].second * sizeof(Number),
355 MPI_BYTE,
356 import_targets_data[i].first,
357 mpi_tag,
358 communicator,
359 &requests[i]);
360 AssertThrowMPI(ierr);
361 temp_array_ptr += import_targets_data[i].second;
362 }
363
364 // initiate the send operations
365
366 // in case we want to import only from a subset of the ghosts we want to
367 // move the data to send to the front of the array
368 AssertIndexRange(n_ghost_indices(), n_ghost_indices_in_larger_set + 1);
369 Number *ghost_array_ptr = ghost_array.data();
370 for (unsigned int i = 0; i < n_ghost_targets; ++i)
371 {
372 // in case we only sent a subset of indices, we now need to move the
373 // data to the correct positions and delete the old content
374 if (n_ghost_indices_in_larger_set > n_ghost_indices() &&
375 ghost_array.size() == n_ghost_indices_in_larger_set)
376 {
377 std::vector<std::pair<unsigned int, unsigned int>>::const_iterator
378 my_ghosts = ghost_indices_subset_data.begin() +
379 ghost_indices_subset_chunks_by_rank_data[i],
380 end_my_ghosts = ghost_indices_subset_data.begin() +
381 ghost_indices_subset_chunks_by_rank_data[i + 1];
382 unsigned int offset = 0;
383 for (; my_ghosts != end_my_ghosts; ++my_ghosts)
384 {
385 const unsigned int chunk_size =
386 my_ghosts->second - my_ghosts->first;
387 if (ghost_array_ptr + offset !=
388 ghost_array.data() + my_ghosts->first)
389 {
390 if constexpr (std::is_same_v<MemorySpaceType,
391 MemorySpace::Host>)
392 {
393 if (offset > my_ghosts->first)
394 std::copy_backward(ghost_array.data() +
395 my_ghosts->first,
396 ghost_array_ptr +
397 my_ghosts->second,
398 ghost_array.data() + offset);
399 else
400 std::copy(ghost_array.data() + my_ghosts->first,
401 ghost_array.data() + my_ghosts->second,
402 ghost_array_ptr + offset);
403 std::fill(
404 std::max(ghost_array.data() + my_ghosts->first,
405 ghost_array_ptr + offset + chunk_size),
406 ghost_array.data() + my_ghosts->second,
407 Number{});
408 }
409 else
410 {
411 Kokkos::View<Number *,
412 MemorySpace::Default::kokkos_space>
413 copy("copy", chunk_size);
414 Kokkos::deep_copy(
415 copy,
416 Kokkos::View<Number *,
417 MemorySpace::Default::kokkos_space>(
418 ghost_array.data() + my_ghosts->first,
419 chunk_size));
420 Kokkos::deep_copy(
421 Kokkos::View<Number *,
422 MemorySpace::Default::kokkos_space>(
423 ghost_array_ptr + offset, chunk_size),
424 copy);
425 Kokkos::deep_copy(
426 Kokkos::View<Number *,
427 MemorySpace::Default::kokkos_space>(
428 std::max(ghost_array.data() + my_ghosts->first,
429 ghost_array_ptr + offset + chunk_size),
430 (ghost_array.data() + my_ghosts->second -
431 std::max(ghost_array.data() + my_ghosts->first,
432 ghost_array_ptr + offset +
433 chunk_size))),
434 0);
435 }
436 }
437 offset += chunk_size;
438 }
439 AssertDimension(offset, ghost_targets_data[i].second);
440 }
441
442 AssertThrow(
443 static_cast<std::size_t>(ghost_targets_data[i].second) *
444 sizeof(Number) <
445 static_cast<std::size_t>(std::numeric_limits<int>::max()),
446 ExcMessage("Index overflow: Maximum message size in MPI is 2GB. "
447 "The number of ghost entries times the size of 'Number' "
448 "exceeds this value. This is not supported."));
449 if (std::is_same_v<MemorySpaceType, MemorySpace::Default>)
450 Kokkos::fence();
451 const int ierr =
452 MPI_Isend(ghost_array_ptr,
453 ghost_targets_data[i].second * sizeof(Number),
454 MPI_BYTE,
455 ghost_targets_data[i].first,
456 mpi_tag,
457 communicator,
458 &requests[n_import_targets + i]);
459 AssertThrowMPI(ierr);
460
461 ghost_array_ptr += ghost_targets_data[i].second;
462 }
463 }
464
465
466
467 namespace internal
468 {
469 // In the import_from_ghosted_array_finish we need to invoke abs() also
470 // on unsigned data types, which is ill-formed on newer C++
471 // standards. To avoid this, we use std::abs on default types but
472 // simply return the number on unsigned types
473 template <typename Number>
474 std::enable_if_t<!std::is_unsigned_v<Number>,
475 typename numbers::NumberTraits<Number>::real_type>
476 get_abs(const Number a)
477 {
478 return std::abs(a);
479 }
480
481 template <typename Number>
482 std::enable_if_t<std::is_unsigned_v<Number>, Number>
483 get_abs(const Number a)
484 {
485 return a;
486 }
487
488 // In the import_from_ghosted_array_finish we might need to calculate the
489 // maximal and minimal value for the given number type, which is not
490 // straight forward for complex numbers. Therefore, comparison of complex
491 // numbers is prohibited and throws an exception.
492 template <typename Number>
493 DEAL_II_HOST_DEVICE Number
494 get_min(const Number a, const Number b)
495 {
496 return std::min(a, b);
497 }
498
499 template <typename Number>
500 std::complex<Number>
501 get_min(const std::complex<Number> a, const std::complex<Number>)
502 {
503 AssertThrow(false,
504 ExcMessage("VectorOperation::min not "
505 "implemented for complex numbers"));
506 return a;
507 }
508
509 template <typename Number>
510 DEAL_II_HOST_DEVICE Number
511 get_max(const Number a, const Number b)
512 {
513 return std::max(a, b);
514 }
515
516 template <typename Number>
517 std::complex<Number>
518 get_max(const std::complex<Number> a, const std::complex<Number>)
519 {
520 AssertThrow(false,
521 ExcMessage("VectorOperation::max not "
522 "implemented for complex numbers"));
523 return a;
524 }
525 } // namespace internal
526
527
528
529 template <typename Number, typename MemorySpaceType>
530 void
531 Partitioner::import_from_ghosted_array_finish(
532 const VectorOperation::values vector_operation,
533 const ArrayView<const Number, MemorySpaceType> &temporary_storage,
534 const ArrayView<Number, MemorySpaceType> &locally_owned_array,
535 const ArrayView<Number, MemorySpaceType> &ghost_array,
536 std::vector<MPI_Request> &requests) const
537 {
538 AssertDimension(temporary_storage.size(), n_import_indices());
539 Assert(ghost_array.size() == n_ghost_indices() ||
540 ghost_array.size() == n_ghost_indices_in_larger_set,
541 ExcGhostIndexArrayHasWrongSize(ghost_array.size(),
542 n_ghost_indices(),
543 n_ghost_indices_in_larger_set));
544
545 // in optimized mode, no communication was started, so leave the
546 // function directly (and only clear ghosts)
547 if constexpr (running_in_debug_mode() == false)
548 if (vector_operation == VectorOperation::insert)
549 {
550 Assert(requests.empty(),
551 ExcInternalError(
552 "Did not expect a non-empty communication "
553 "request when inserting. Check that the same "
554 "vector_operation argument was passed to "
555 "import_from_ghosted_array_start as is passed "
556 "to import_from_ghosted_array_finish."));
557
558 Kokkos::deep_copy(
559 Kokkos::View<Number *, typename MemorySpaceType::kokkos_space>(
560 ghost_array.data(), ghost_array.size()),
561 0);
562 return;
563 }
564
565 // nothing to do when we neither have import nor ghost indices.
566 if (n_ghost_indices() == 0 && n_import_indices() == 0)
567 return;
568
569 const unsigned int n_import_targets = import_targets_data.size();
570 const unsigned int n_ghost_targets = ghost_targets_data.size();
571
572 # if defined(DEAL_II_MPI_WITH_DEVICE_SUPPORT)
573 // When using device-aware MPI, the set of local indices that are ghosts
574 // indices on other processors is expanded in arrays. This is for
575 // performance reasons as this can significantly decrease the number of
576 // kernel launched. The indices are expanded the first time the function
577 // is called.
578 if ((std::is_same_v<MemorySpaceType, MemorySpace::Default>)&&(
579 import_indices_plain_dev.empty()))
580 initialize_import_indices_plain_dev();
581 # endif
582
583 if (vector_operation != VectorOperation::insert)
584 AssertDimension(n_ghost_targets + n_import_targets, requests.size());
585 // first wait for the receive to complete
586 if (requests.size() > 0 && n_import_targets > 0)
587 {
588 AssertDimension(locally_owned_array.size(), locally_owned_size());
589 const int ierr =
590 MPI_Waitall(n_import_targets, requests.data(), MPI_STATUSES_IGNORE);
591 AssertThrowMPI(ierr);
592
593 const Number *read_position = temporary_storage.data();
594 # if defined(DEAL_II_MPI_WITH_DEVICE_SUPPORT)
595 if constexpr (std::is_same_v<MemorySpaceType, MemorySpace::Default>)
596 {
597 if (vector_operation == VectorOperation::add)
598 {
599 for (const auto &import_indices_plain :
600 import_indices_plain_dev)
601 {
602 const auto chunk_size = import_indices_plain.size();
603
604 using IndexType = decltype(chunk_size);
605 MemorySpace::Default::kokkos_space::execution_space exec;
606 Kokkos::parallel_for(
607 "dealii::fill locally_owned_array, add",
608 Kokkos::RangePolicy<
609 MemorySpace::Default::kokkos_space::execution_space>(
610 exec, 0, chunk_size),
611 KOKKOS_LAMBDA(IndexType idx) {
612 locally_owned_array
613 .data()[import_indices_plain(idx)] +=
614 read_position[idx];
615 });
616 exec.fence();
617
618 read_position += chunk_size;
619 }
620 }
621 else if (vector_operation == VectorOperation::min)
622 {
623 for (const auto &import_indices_plain :
624 import_indices_plain_dev)
625 {
626 const auto chunk_size = import_indices_plain.size();
627
628 using IndexType = decltype(chunk_size);
629 MemorySpace::Default::kokkos_space::execution_space exec;
630 Kokkos::parallel_for(
631 "dealii::fill locally_owned_array, min",
632 Kokkos::RangePolicy<
633 MemorySpace::Default::kokkos_space::execution_space>(
634 exec, 0, chunk_size),
635 KOKKOS_LAMBDA(IndexType idx) {
636 locally_owned_array
637 .data()[import_indices_plain(idx)] =
638 internal::get_min(
639 locally_owned_array
640 .data()[import_indices_plain(idx)],
641 read_position[idx]);
642 });
643 exec.fence();
644
645 read_position += chunk_size;
646 }
647 }
648 else if (vector_operation == VectorOperation::max)
649 {
650 for (const auto &import_indices_plain :
651 import_indices_plain_dev)
652 {
653 const auto chunk_size = import_indices_plain.size();
654
655 using IndexType = decltype(chunk_size);
656 MemorySpace::Default::kokkos_space::execution_space exec;
657 Kokkos::parallel_for(
658 "dealii::fill locally_owned_array, max",
659 Kokkos::RangePolicy<
660 MemorySpace::Default::kokkos_space::execution_space>(
661 exec, 0, chunk_size),
662 KOKKOS_LAMBDA(IndexType idx) {
663 locally_owned_array
664 .data()[import_indices_plain(idx)] =
665 internal::get_max(
666 locally_owned_array
667 .data()[import_indices_plain(idx)],
668 read_position[idx]);
669 });
670 exec.fence();
671
672 read_position += chunk_size;
673 }
674 }
675 else
676 {
677 for (const auto &import_indices_plain :
678 import_indices_plain_dev)
679 {
680 // We can't easily assert here, so we just move the
681 // pointer matching the host code.
682 const auto chunk_size = import_indices_plain.size();
683 read_position += chunk_size;
684 }
685 }
686 }
687 else
688 # endif
689 {
690 // If the operation is no insertion, add the imported data to the
691 // local values. For insert, nothing is done here (but in debug
692 // mode we assert that the specified value is either zero or
693 // matches with the ones already present
694 if (vector_operation == VectorOperation::add)
695 for (const auto &import_range : import_indices_data)
696 for (unsigned int j = import_range.first;
697 j < import_range.second;
698 j++)
699 locally_owned_array[j] += *read_position++;
700 else if (vector_operation == VectorOperation::min)
701 for (const auto &import_range : import_indices_data)
702 for (unsigned int j = import_range.first;
703 j < import_range.second;
704 j++)
705 {
706 locally_owned_array[j] =
707 internal::get_min(*read_position,
708 locally_owned_array[j]);
709 ++read_position;
710 }
711 else if (vector_operation == VectorOperation::max)
712 for (const auto &import_range : import_indices_data)
713 for (unsigned int j = import_range.first;
714 j < import_range.second;
715 j++)
716 {
717 locally_owned_array[j] =
718 internal::get_max(*read_position,
719 locally_owned_array[j]);
720 ++read_position;
721 }
722 else
723 for (const auto &import_range : import_indices_data)
724 for (unsigned int j = import_range.first;
725 j < import_range.second;
726 j++, read_position++)
727 // Below we use relatively large precision in units in the
728 // last place (ULP) as this Assert can be easily triggered
729 // in p::d::SolutionTransfer. The rationale is that during
730 // interpolation on two elements sharing the face, values on
731 // this face obtained from each side might be different due
732 // to additions being done in different order. If the local
733 // value is zero, it indicates that the local process has
734 // not set the value during the cell loop and its value can
735 // be safely overridden.
736 Assert(
737 *read_position == Number() ||
738 internal::get_abs(locally_owned_array[j] -
739 *read_position) <=
740 internal::get_abs(locally_owned_array[j] +
741 *read_position) *
742 100000. *
743 std::numeric_limits<typename numbers::NumberTraits<
744 Number>::real_type>::epsilon(),
745 typename dealii::LinearAlgebra::distributed::Vector<
746 Number>::ExcNonMatchingElements(*read_position,
747 locally_owned_array[j],
748 my_pid));
749 }
750
751 AssertDimension(read_position - temporary_storage.data(),
752 n_import_indices());
753 }
754
755 // wait for the send operations to complete
756 if (requests.size() > 0 && n_ghost_targets > 0)
757 {
758 const int ierr = MPI_Waitall(n_ghost_targets,
759 &requests[n_import_targets],
760 MPI_STATUSES_IGNORE);
761 AssertThrowMPI(ierr);
762 }
763 else
764 AssertDimension(n_ghost_indices(), 0);
765
766 // clear the ghost array in case we did not yet do that in the _start
767 // function
768 if (ghost_array.size() > 0)
769 {
770 Assert(ghost_array.begin() != nullptr, ExcInternalError());
771
772 # if defined(DEAL_II_MPI_WITH_DEVICE_SUPPORT)
773 if constexpr (std::is_same_v<MemorySpaceType, MemorySpace::Default>)
774 {
775 Kokkos::deep_copy(
776 Kokkos::View<Number *, MemorySpace::Default::kokkos_space>(
777 ghost_array.data(), n_ghost_indices()),
778 Number(0));
779 }
780 else
781 # endif
782 {
783 std::fill(ghost_array.data(),
784 ghost_array.data() + n_ghost_indices(),
785 Number(0));
786 }
787 }
788
789 // clear the compress requests
790 requests.resize(0);
791 }
792
793
794 # endif // ifdef DEAL_II_WITH_MPI
795 #endif // ifndef DOXYGEN
796
797 } // end of namespace MPI
798
799 } // namespace Utilities
800
801
802 DEAL_II_NAMESPACE_CLOSE
803
804 #endif