Debian Codegraph @

dune/istl/owneroverlapcopy.hh

raw
1 // SPDX-FileCopyrightText: Copyright © DUNE Project contributors, see file LICENSE.md in module root
2 // SPDX-License-Identifier: LicenseRef-GPL-2.0-only-with-DUNE-exception
3 // -*- tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 2 -*-
4 // vi: set et ts=4 sw=2 sts=2:
5 #ifndef DUNE_ISTL_OWNEROVERLAPCOPY_HH
6 #define DUNE_ISTL_OWNEROVERLAPCOPY_HH
7
8 #include <new>
9 #include <iostream>
10 #include <vector>
11 #include <list>
12 #include <map>
13 #include <set>
14 #include <tuple>
15
16 #include <cmath>
17
18 // MPI header
19 #if HAVE_MPI
20 #include <mpi.h>
21 #endif
22
23 #include <dune/common/enumset.hh>
24
25 #if HAVE_MPI
26 #include <dune/common/parallel/indexset.hh>
27 #include <dune/common/parallel/communicator.hh>
28 #include <dune/common/parallel/remoteindices.hh>
29 #include <dune/common/parallel/mpicommunication.hh>
30 #endif
31
32 #include "solvercategory.hh"
33 #include "istlexception.hh"
34 #include <dune/common/parallel/communication.hh>
35 #include <dune/istl/matrixmarket.hh>
36
37 template<int dim, template<class,class> class Comm>
38 void testRedistributed(int s);
39
40
41 namespace Dune {
42
43 /**
44 @addtogroup ISTL_Comm
45 @{
46 */
47
48 /**
49 * @file
50 * @brief Classes providing communication interfaces for
51 * overlapping Schwarz methods.
52 * @author Peter Bastian
53 */
54
55 /**
56 * @brief Attribute set for overlapping Schwarz.
57 */
58 struct OwnerOverlapCopyAttributeSet
59 {
60 enum AttributeSet {
61 owner=1, overlap=2, copy=3
62 };
63 };
64
65 /**
66 * @brief Information about the index distribution.
67 *
68 * This class contains information about indices local to
69 * the process together with information about on which
70 * processes those indices are also present together with the
71 * attribute they have there.
72 *
73 * This information might be used to set up an IndexSet together with
74 * an RemoteIndices object needed for the ISTL communication classes.
75 */
76 template <class G, class L>
77 class IndexInfoFromGrid
78 {
79 public:
80 /** @brief The type of the global index. */
81 typedef G GlobalIdType;
82
83 /** @brief The type of the local index. */
84 typedef L LocalIdType;
85
86 /**
87 * @brief A triple describing a local index.
88 *
89 * The triple consists of the global index and the local
90 * index and an attribute
91 */
92 typedef std::tuple<GlobalIdType,LocalIdType,int> IndexTripel;
93 /**
94 * @brief A triple describing a remote index.
95 *
96 * The triple consists of a process number and the global index and
97 * the attribute of the index at the remote process.
98 */
99 typedef std::tuple<int,GlobalIdType,int> RemoteIndexTripel;
100
101 /**
102 * @brief Add a new index triple to the set of local indices.
103 *
104 * @param x The index triple.
105 */
106 void addLocalIndex (const IndexTripel& x)
107 {
108 if (std::get<2>(x)!=OwnerOverlapCopyAttributeSet::owner &&
109 std::get<2>(x)!=OwnerOverlapCopyAttributeSet::overlap &&
110 std::get<2>(x)!=OwnerOverlapCopyAttributeSet::copy)
111 DUNE_THROW(ISTLError,"OwnerOverlapCopyCommunication: global index not in index set");
112 localindices.insert(x);
113 }
114
115 /**
116 * @brief Add a new remote index triple to the set of remote indices.
117 *
118 * @param x The index triple to add.
119 */
120 void addRemoteIndex (const RemoteIndexTripel& x)
121 {
122 if (std::get<2>(x)!=OwnerOverlapCopyAttributeSet::owner &&
123 std::get<2>(x)!=OwnerOverlapCopyAttributeSet::overlap &&
124 std::get<2>(x)!=OwnerOverlapCopyAttributeSet::copy)
125 DUNE_THROW(ISTLError,"OwnerOverlapCopyCommunication: global index not in index set");
126 remoteindices.insert(x);
127 }
128
129 /**
130 * @brief Get the set of indices local to the process.
131 * @return The set of local indices.
132 */
133 const std::set<IndexTripel>& localIndices () const
134 {
135 return localindices;
136 }
137
138 /**
139 * @brief Get the set of remote indices.
140 * @return the set of remote indices.
141 */
142 const std::set<RemoteIndexTripel>& remoteIndices () const
143 {
144 return remoteindices;
145 }
146
147 /**
148 * @brief Remove all indices from the sets.
149 */
150 void clear ()
151 {
152 localindices.clear();
153 remoteindices.clear();
154 }
155
156 private:
157 /** @brief The set of local indices. */
158 std::set<IndexTripel> localindices;
159 /** @brief The set of remote indices. */
160 std::set<RemoteIndexTripel> remoteindices;
161 };
162
163
164 #if HAVE_MPI
165
166 /**
167 * @brief A class setting up standard communication for a two-valued
168 * attribute set with owner/overlap/copy semantics.
169 *
170 * set up communication from known distribution with owner/overlap/copy semantics
171 */
172 template <class GlobalIdType, class LocalIdType=int>
173 class OwnerOverlapCopyCommunication
174 {
175 template<typename M, typename G, typename L>
176 friend void loadMatrixMarket(M&,
177 const std::string&,
178 OwnerOverlapCopyCommunication<G,L>&,
179 bool);
180 // used types
181 typedef typename IndexInfoFromGrid<GlobalIdType,LocalIdType>::IndexTripel IndexTripel;
182 typedef typename IndexInfoFromGrid<GlobalIdType,LocalIdType>::RemoteIndexTripel RemoteIndexTripel;
183 typedef typename std::set<IndexTripel>::const_iterator localindex_iterator;
184 typedef typename std::set<RemoteIndexTripel>::const_iterator remoteindex_iterator;
185 typedef typename OwnerOverlapCopyAttributeSet::AttributeSet AttributeSet;
186 typedef Dune::ParallelLocalIndex<AttributeSet> LI;
187 public:
188 typedef Dune::ParallelIndexSet<GlobalIdType,LI,512> PIS;
189 typedef Dune::RemoteIndices<PIS> RI;
190 typedef Dune::RemoteIndexListModifier<PIS,typename RI::Allocator,false> RILM;
191 typedef typename RI::RemoteIndex RX;
192 typedef Dune::BufferedCommunicator BC;
193 typedef Dune::Interface IF;
194 typedef EnumItem<AttributeSet,OwnerOverlapCopyAttributeSet::owner> OwnerSet;
195 typedef EnumItem<AttributeSet,OwnerOverlapCopyAttributeSet::copy> CopySet;
196 typedef Combine<EnumItem<AttributeSet,OwnerOverlapCopyAttributeSet::owner>,EnumItem<AttributeSet,OwnerOverlapCopyAttributeSet::overlap>,AttributeSet> OwnerOverlapSet;
197 typedef Dune::AllSet<AttributeSet> AllSet;
198 protected:
199
200
201 /** \brief gather/scatter callback for communication */
202 template<typename T>
203 struct CopyGatherScatter
204 {
205 typedef typename CommPolicy<T>::IndexedType V;
206
207 static V gather(const T& a, std::size_t i)
208 {
209 return a[i];
210 }
211
212 static void scatter(T& a, V v, std::size_t i)
213 {
214 a[i] = v;
215 }
216 };
217 template<typename T>
218 struct AddGatherScatter
219 {
220 typedef typename CommPolicy<T>::IndexedType V;
221
222 static V gather(const T& a, std::size_t i)
223 {
224 return a[i];
225 }
226
227 static void scatter(T& a, V v, std::size_t i)
228 {
229 a[i] += v;
230 }
231 };
232
233 void buildOwnerOverlapToAllInterface () const
234 {
235 if (OwnerOverlapToAllInterfaceBuilt)
236 OwnerOverlapToAllInterface.free();
237 OwnerOverlapSet sourceFlags;
238 Combine<OwnerOverlapSet,EnumItem<AttributeSet,OwnerOverlapCopyAttributeSet::copy>,AttributeSet>
239 destFlags;
240 OwnerOverlapToAllInterface.build(ri,sourceFlags,destFlags);
241 OwnerOverlapToAllInterfaceBuilt = true;
242 }
243
244 void buildOwnerToAllInterface () const
245 {
246 if (OwnerToAllInterfaceBuilt)
247 OwnerToAllInterface.free();
248 OwnerSet sourceFlags;
249 AllSet destFlags;
250 OwnerToAllInterface.build(ri,sourceFlags,destFlags);
251 OwnerToAllInterfaceBuilt = true;
252 }
253
254 void buildOwnerCopyToAllInterface () const
255 {
256 if (OwnerCopyToAllInterfaceBuilt)
257 OwnerCopyToAllInterface.free();
258
259 typedef Combine<EnumItem<AttributeSet,OwnerOverlapCopyAttributeSet::owner>,EnumItem<AttributeSet,OwnerOverlapCopyAttributeSet::copy>,AttributeSet> OwnerCopySet;
260 OwnerCopySet sourceFlags;
261 Combine<OwnerCopySet,EnumItem<AttributeSet,OwnerOverlapCopyAttributeSet::overlap>,AttributeSet> destFlags;
262 OwnerCopyToAllInterface.build(ri,sourceFlags,destFlags);
263 OwnerCopyToAllInterfaceBuilt = true;
264 }
265
266 void buildOwnerCopyToOwnerCopyInterface () const
267 {
268 if (OwnerCopyToOwnerCopyInterfaceBuilt)
269 OwnerCopyToOwnerCopyInterface.free();
270
271
272 typedef Combine<EnumItem<AttributeSet,OwnerOverlapCopyAttributeSet::owner>,EnumItem<AttributeSet,OwnerOverlapCopyAttributeSet::copy>,AttributeSet> OwnerCopySet;
273 OwnerCopySet sourceFlags;
274 OwnerCopySet destFlags;
275 OwnerCopyToOwnerCopyInterface.build(ri,sourceFlags,destFlags);
276 OwnerCopyToOwnerCopyInterfaceBuilt = true;
277 }
278
279 void buildCopyToAllInterface () const
280 {
281 if (CopyToAllInterfaceBuilt)
282 CopyToAllInterface.free();
283 CopySet sourceFlags;
284 AllSet destFlags;
285 CopyToAllInterface.build(ri,sourceFlags,destFlags);
286 CopyToAllInterfaceBuilt = true;
287 }
288
289 public:
290
291 /**
292 * @brief Get Solver Category.
293 * @return The Solver Category.
294 */
295 SolverCategory::Category category () const {
296 return category_;
297 }
298
299 const Communication<MPI_Comm>& communicator() const
300 {
301 return cc;
302 }
303
304 /**
305 * @brief Communicate values from owner data points to all other data points.
306 *
307 * @brief source The data to send from.
308 * @brief dest The data to send to.
309 */
310 template<class T>
311 void copyOwnerToAll (const T& source, T& dest) const
312 {
313 if (!OwnerToAllInterfaceBuilt)
314 buildOwnerToAllInterface ();
315 BC communicator;
316 communicator.template build<T>(OwnerToAllInterface);
317 communicator.template forward<CopyGatherScatter<T> >(source,dest);
318 communicator.free();
319 }
320
321 /**
322 * @brief Communicate values from copy data points to all other data points.
323 *
324 * @brief source The data to send from.
325 * @brief dest The data to send to.
326 */
327 template<class T>
328 void copyCopyToAll (const T& source, T& dest) const
329 {
330 if (!CopyToAllInterfaceBuilt)
331 buildCopyToAllInterface ();
332 BC communicator;
333 communicator.template build<T>(CopyToAllInterface);
334 communicator.template forward<CopyGatherScatter<T> >(source,dest);
335 communicator.free();
336 }
337
338 /**
339 * @brief Communicate values from owner data points to all other data points and add them to those values.
340 *
341 * @brief source The data to send from.
342 * @brief dest The data to add them communicated values to.
343 */
344 template<class T>
345 void addOwnerOverlapToAll (const T& source, T& dest) const
346 {
347 if (!OwnerOverlapToAllInterfaceBuilt)
348 buildOwnerOverlapToAllInterface ();
349 BC communicator;
350 communicator.template build<T>(OwnerOverlapToAllInterface);
351 communicator.template forward<AddGatherScatter<T> >(source,dest);
352 communicator.free();
353 }
354
355 /**
356 * @brief Communicate values from owner and copy data points to all other data points and add them to those values.
357 *
358 * @brief source The data to send from.
359 * @brief dest The data to add them communicated values to.
360 */
361 template<class T>
362 void addOwnerCopyToAll (const T& source, T& dest) const
363 {
364 if (!OwnerCopyToAllInterfaceBuilt)
365 buildOwnerCopyToAllInterface ();
366 BC communicator;
367 communicator.template build<T>(OwnerCopyToAllInterface);
368 communicator.template forward<AddGatherScatter<T> >(source,dest);
369 communicator.free();
370 }
371
372 /**
373 * @brief Communicate values from owner and copy data points to owner and copy data points and add them to those values.
374 *
375 * @brief source The data to send from.
376 * @brief dest The data to add the communicated values to.
377 */
378 template<class T>
379 void addOwnerCopyToOwnerCopy (const T& source, T& dest) const
380 {
381 if (!OwnerCopyToOwnerCopyInterfaceBuilt)
382 buildOwnerCopyToOwnerCopyInterface ();
383 BC communicator;
384 communicator.template build<T>(OwnerCopyToOwnerCopyInterface);
385 communicator.template forward<AddGatherScatter<T> >(source,dest);
386 communicator.free();
387 }
388
389
390 /**
391 * @brief Compute a global dot product of two vectors.
392 *
393 * @param x The first vector of the product.
394 * @param y The second vector of the product.
395 * @param result Reference to store the result in.
396 */
397 template<class T1, class T2>
398 void dot (const T1& x, const T1& y, T2& result) const
399 {
400 using real_type = typename FieldTraits<typename T1::field_type>::real_type;
401 // set up mask vector
402 if (mask.size()!=static_cast<typename std::vector<double>::size_type>(x.size()))
403 {
404 mask.resize(x.size());
405 for (typename std::vector<double>::size_type i=0; i<mask.size(); i++)
406 mask[i] = 1;
407 for (typename PIS::const_iterator i=pis.begin(); i!=pis.end(); ++i)
408 if (i->local().attribute()!=OwnerOverlapCopyAttributeSet::owner)
409 mask[i->local().local()] = 0;
410 }
411 result = T2(0.0);
412
413 for (typename T1::size_type i=0; i<x.size(); i++)
414 result += (x[i]*(y[i]))*static_cast<real_type>(mask[i]);
415 result = cc.sum(result);
416 }
417
418 /**
419 * @brief Compute the global Euclidean norm of a vector.
420 *
421 * @param x The vector to compute the norm of.
422 * @return The global Euclidean norm of that vector.
423 */
424 template<class T1>
425 typename FieldTraits<typename T1::field_type>::real_type norm (const T1& x) const
426 {
427 using real_type = typename FieldTraits<typename T1::field_type>::real_type;
428
429 // set up mask vector
430 if (mask.size()!=static_cast<typename std::vector<double>::size_type>(x.size()))
431 {
432 mask.resize(x.size());
433 for (typename std::vector<double>::size_type i=0; i<mask.size(); i++)
434 mask[i] = 1;
435 for (typename PIS::const_iterator i=pis.begin(); i!=pis.end(); ++i)
436 if (i->local().attribute()!=OwnerOverlapCopyAttributeSet::owner)
437 mask[i->local().local()] = 0;
438 }
439 auto result = real_type(0.0);
440 for (typename T1::size_type i=0; i<x.size(); i++)
441 result += Impl::asVector(x[i]).two_norm2()*mask[i];
442 using std::sqrt;
443 return sqrt(cc.sum(result));
444 }
445
446 typedef Dune::EnumItem<AttributeSet,OwnerOverlapCopyAttributeSet::copy> CopyFlags;
447
448 /** @brief The type of the parallel index set. */
449 typedef Dune::ParallelIndexSet<GlobalIdType,LI,512> ParallelIndexSet;
450
451 /** @brief The type of the remote indices. */
452 typedef Dune::RemoteIndices<PIS> RemoteIndices;
453
454 /**
455 * @brief The type of the reverse lookup of indices. */
456 typedef Dune::GlobalLookupIndexSet<ParallelIndexSet> GlobalLookupIndexSet;
457
458 /**
459 * @brief Get the underlying parallel index set.
460 * @return The underlying parallel index set.
461 */
462 const ParallelIndexSet& indexSet() const
463 {
464 return pis;
465 }
466
467 /**
468 * @brief Get the underlying remote indices.
469 * @return The underlying remote indices.
470 */
471 const RemoteIndices& remoteIndices() const
472 {
473 return ri;
474 }
475
476 /**
477 * @brief Get the underlying parallel index set.
478 * @return The underlying parallel index set.
479 */
480 ParallelIndexSet& indexSet()
481 {
482 return pis;
483 }
484
485
486 /**
487 * @brief Get the underlying remote indices.
488 * @return The underlying remote indices.
489 */
490 RemoteIndices& remoteIndices()
491 {
492 return ri;
493 }
494
495 void buildGlobalLookup()
496 {
497 if(globalLookup_) {
498 if(pis.seqNo()==oldseqNo)
499 // Nothing changed!
500 return;
501 delete globalLookup_;
502 }
503
504 globalLookup_ = new GlobalLookupIndexSet(pis);
505 oldseqNo = pis.seqNo();
506 }
507
508 void buildGlobalLookup(std::size_t size)
509 {
510 if(globalLookup_) {
511 if(pis.seqNo()==oldseqNo)
512 // Nothing changed!
513 return;
514 delete globalLookup_;
515 }
516 globalLookup_ = new GlobalLookupIndexSet(pis, size);
517 oldseqNo = pis.seqNo();
518 }
519
520 void freeGlobalLookup()
521 {
522 delete globalLookup_;
523 globalLookup_=0;
524 }
525
526 const GlobalLookupIndexSet& globalLookup() const
527 {
528 assert(globalLookup_ != 0);
529 return *globalLookup_;
530 }
531
532 /**
533 * @brief Set vector to zero at copy dofs
534 *
535 * @param x The vector to project.
536 */
537 template<class T1>
538 void project (T1& x) const
539 {
540 for (typename PIS::const_iterator i=pis.begin(); i!=pis.end(); ++i)
541 if (i->local().attribute()==OwnerOverlapCopyAttributeSet::copy)
542 x[i->local().local()] = 0;
543 }
544
545 /**
546 * @brief Construct the communication without any indices.
547 *
548 * The local index set and the remote indices have to be set up
549 * later on.
550 * @param comm_ The MPI Communicator to use, e. g. MPI_COMM_WORLD
551 * @param cat_ The Solver category, default is overlapping
552 * @param freecomm_ Whether to free the communicator comm_ in the destructor, default is false
553 */
554 OwnerOverlapCopyCommunication (MPI_Comm comm_,
555 SolverCategory::Category cat_ = SolverCategory::overlapping,
556 bool freecomm_ = false)
557 : comm(comm_), cc(comm_), pis(), ri(pis,pis,comm_),
558 OwnerToAllInterfaceBuilt(false), OwnerOverlapToAllInterfaceBuilt(false),
559 OwnerCopyToAllInterfaceBuilt(false), OwnerCopyToOwnerCopyInterfaceBuilt(false),
560 CopyToAllInterfaceBuilt(false), globalLookup_(0), category_(cat_),
561 freecomm(freecomm_)
562 {}
563
564 /**
565 * @brief Construct the communication without any indices using MPI_COMM_WORLD.
566 *
567 * The local index set and the remote indices have to be set up
568 * later on.
569 * @param cat_ The Solver category, default is overlapping
570 is false
571 */
572 OwnerOverlapCopyCommunication (SolverCategory::Category cat_ = SolverCategory::overlapping)
573 : comm(MPI_COMM_WORLD), cc(MPI_COMM_WORLD), pis(), ri(pis,pis,MPI_COMM_WORLD),
574 OwnerToAllInterfaceBuilt(false), OwnerOverlapToAllInterfaceBuilt(false),
575 OwnerCopyToAllInterfaceBuilt(false), OwnerCopyToOwnerCopyInterfaceBuilt(false),
576 CopyToAllInterfaceBuilt(false), globalLookup_(0), category_(cat_), freecomm(false)
577 {}
578
579 /**
580 * @brief Constructor
581 * @param indexinfo The set of IndexTripels describing the local and remote indices.
582 * @param comm_ The communicator to use in the communication.
583 * @param cat_ The Solver category, default is overlapping
584 * @param freecomm_ Whether to free the communicator comm_ in the destructor, default is false
585 */
586 OwnerOverlapCopyCommunication (const IndexInfoFromGrid<GlobalIdType, LocalIdType>& indexinfo,
587 MPI_Comm comm_,
588 SolverCategory::Category cat_ = SolverCategory::overlapping,
589 bool freecomm_ = false)
590 : comm(comm_), cc(comm_), OwnerToAllInterfaceBuilt(false),
591 OwnerOverlapToAllInterfaceBuilt(false), OwnerCopyToAllInterfaceBuilt(false),
592 OwnerCopyToOwnerCopyInterfaceBuilt(false), CopyToAllInterfaceBuilt(false),
593 globalLookup_(0), category_(cat_), freecomm(freecomm_)
594 {
595 // set up an ISTL index set
596 pis.beginResize();
597 for (localindex_iterator i=indexinfo.localIndices().begin(); i!=indexinfo.localIndices().end(); ++i)
598 {
599 if (std::get<2>(*i)==OwnerOverlapCopyAttributeSet::owner)
600 pis.add(std::get<0>(*i),LI(std::get<1>(*i),OwnerOverlapCopyAttributeSet::owner,true));
601 if (std::get<2>(*i)==OwnerOverlapCopyAttributeSet::overlap)
602 pis.add(std::get<0>(*i),LI(std::get<1>(*i),OwnerOverlapCopyAttributeSet::overlap,true));
603 if (std::get<2>(*i)==OwnerOverlapCopyAttributeSet::copy)
604 pis.add(std::get<0>(*i),LI(std::get<1>(*i),OwnerOverlapCopyAttributeSet::copy,true));
605 // std::cout << cc.rank() << ": adding index " << std::get<0>(*i) << " " << std::get<1>(*i) << " " << std::get<2>(*i) << std::endl;
606 }
607 pis.endResize();
608
609 // build remote indices WITHOUT communication
610 // std::cout << cc.rank() << ": build remote indices" << std::endl;
611 ri.setIndexSets(pis,pis,cc);
612 if (indexinfo.remoteIndices().size()>0)
613 {
614 remoteindex_iterator i=indexinfo.remoteIndices().begin();
615 int p = std::get<0>(*i);
616 RILM modifier = ri.template getModifier<false,true>(p);
617 typename PIS::const_iterator pi=pis.begin();
618 for ( ; i!=indexinfo.remoteIndices().end(); ++i)
619 {
620 // handle processor change
621 if (p!=std::get<0>(*i))
622 {
623 p = std::get<0>(*i);
624 modifier = ri.template getModifier<false,true>(p);
625 pi=pis.begin();
626 }
627
628 // position to correct entry in parallel index set
629 while (pi->global()!=std::get<1>(*i) && pi!=pis.end())
630 ++pi;
631 if (pi==pis.end())
632 DUNE_THROW(ISTLError,"OwnerOverlapCopyCommunication: global index not in index set");
633
634 // insert entry
635 // std::cout << cc.rank() << ": adding remote index " << std::get<0>(*i) << " " << std::get<1>(*i) << " " << std::get<2>(*i) << std::endl;
636 if (std::get<2>(*i)==OwnerOverlapCopyAttributeSet::owner)
637 modifier.insert(RX(OwnerOverlapCopyAttributeSet::owner,&(*pi)));
638 if (std::get<2>(*i)==OwnerOverlapCopyAttributeSet::overlap)
639 modifier.insert(RX(OwnerOverlapCopyAttributeSet::overlap,&(*pi)));
640 if (std::get<2>(*i)==OwnerOverlapCopyAttributeSet::copy)
641 modifier.insert(RX(OwnerOverlapCopyAttributeSet::copy,&(*pi)));
642 }
643 }else{
644 // Force remote indices to be synced!
645 ri.template getModifier<false,true>(0);
646 }
647 }
648
649 // destructor: free memory in some objects
650 ~OwnerOverlapCopyCommunication ()
651 {
652 ri.free();
653 if (OwnerToAllInterfaceBuilt) OwnerToAllInterface.free();
654 if (OwnerOverlapToAllInterfaceBuilt) OwnerOverlapToAllInterface.free();
655 if (OwnerCopyToAllInterfaceBuilt) OwnerCopyToAllInterface.free();
656 if (OwnerCopyToOwnerCopyInterfaceBuilt) OwnerCopyToOwnerCopyInterface.free();
657 if (CopyToAllInterfaceBuilt) CopyToAllInterface.free();
658 if (globalLookup_) delete globalLookup_;
659 if (freecomm && (comm != MPI_COMM_NULL))
660 {
661 // If it is possible to query whether MPI_Finalize
662 // was called, only free the communicator before
663 // calling MPI_Finalize.
664 int wasFinalized = 0;
665 MPI_Finalized(&wasFinalized);
666 if (!wasFinalized) {
667 MPI_Comm_free(&comm);
668 }
669 }
670 }
671
672 private:
673 OwnerOverlapCopyCommunication (const OwnerOverlapCopyCommunication&)
674 {}
675 MPI_Comm comm;
676 Communication<MPI_Comm> cc;
677 PIS pis;
678 RI ri;
679 mutable IF OwnerToAllInterface;
680 mutable bool OwnerToAllInterfaceBuilt;
681 mutable IF OwnerOverlapToAllInterface;
682 mutable bool OwnerOverlapToAllInterfaceBuilt;
683 mutable IF OwnerCopyToAllInterface;
684 mutable bool OwnerCopyToAllInterfaceBuilt;
685 mutable IF OwnerCopyToOwnerCopyInterface;
686 mutable bool OwnerCopyToOwnerCopyInterfaceBuilt;
687 mutable IF CopyToAllInterface;
688 mutable bool CopyToAllInterfaceBuilt;
689 mutable std::vector<double> mask;
690 int oldseqNo;
691 GlobalLookupIndexSet* globalLookup_;
692 const SolverCategory::Category category_;
693 bool freecomm;
694 };
695
696 #endif
697
698
699 /** @} end documentation */
700
701 } // end namespace
702
703 #endif