Debian Codegraph @

dune/uggrid/parallel/dddif/trans.cc

raw
1 // SPDX-FileCopyrightText: Copyright © DUNE Project contributors, see file LICENSE.md in module root
2 // SPDX-License-Identifier: LGPL-2.1-or-later
3 // -*- tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 2 -*-
4 // vi: set et ts=4 sw=2 sts=2:
5 /****************************************************************************/
6 /* */
7 /* File: trans.c */
8 /* */
9 /* Purpose: create new grid distribution according to lb-marks of master */
10 /* elements. */
11 /* */
12 /* Author: Klaus Birken, Stefan Lang */
13 /* Institut fuer Computeranwendungen III */
14 /* Universitaet Stuttgart */
15 /* Pfaffenwaldring 27 */
16 /* 70569 Stuttgart */
17 /* email: [email protected] */
18 /* */
19 /* History: 961216 kb begin */
20 /* */
21 /* Remarks: */
22 /* */
23 /****************************************************************************/
24
25
26 #ifdef ModelP
27
28 /****************************************************************************/
29 /* */
30 /* include files */
31 /* system include files */
32 /* application include files */
33 /* */
34 /****************************************************************************/
35
36 #include <config.h>
37 #include <cassert>
38
39 #include <dune/uggrid/parallel/ppif/ppifcontext.hh>
40
41 #include "parallel.h"
42 #include <dune/uggrid/ugdevices.h>
43 #include <dune/uggrid/domain/std_domain.h>
44 #include <dune/uggrid/gm/algebra.h>
45 #include <dune/uggrid/gm/evm.h>
46 #include <dune/uggrid/gm/gm.h>
47 #include <dune/uggrid/gm/refine.h>
48 #include <dune/uggrid/gm/ugm.h>
49 #include <dune/uggrid/low/debug.h>
50 #include <dune/uggrid/low/namespace.h>
51 #include <dune/uggrid/low/ugtypes.h>
52
53 /* UG namespaces: */
54 USING_UG_NAMESPACES
55
56 /* PPIF namespace: */
57 using namespace PPIF;
58
59 /****************************************************************************/
60 /* */
61 /* defines in the following order */
62 /* */
63 /* compile time constants defining static data size (i.e. arrays) */
64 /* other constants */
65 /* macros */
66 /* */
67 /****************************************************************************/
68
69 enum GhostCmds { GC_Keep, GC_ToMaster, GC_Delete };
70
71
72 #define XferElement(context, elem,dest,prio) \
73 { PRINTDEBUG(dddif,1,("%4d: XferElement(): XferCopy elem=" EID_FMTX " dest=%d prio=%d\n", \
74 me,EID_PRTX(elem), dest, prio)); \
75 XFERECOPYX(context, (elem), dest, prio, \
76 (OBJT(elem)==BEOBJ) ? \
77 BND_SIZE_TAG(TAG(elem)) : \
78 INNER_SIZE_TAG(TAG(elem))); }
79
80 #ifdef DDD_PRIO_ENV
81 #undef DDD_PrioritySet
82 #endif
83
84 /****************************************************************************/
85 /* */
86 /* data structures used in this source file (exported data structures are */
87 /* in the corresponding include file!) */
88 /* */
89 /****************************************************************************/
90
91
92 /****************************************************************************/
93 /* */
94 /* definition of exported global variables */
95 /* */
96 /****************************************************************************/
97
98
99 /****************************************************************************/
100 /* */
101 /* definition of variables global to this source file only (static!) */
102 /* */
103 /****************************************************************************/
104
105
106 /****************************************************************************/
107 /* */
108 /* forward declarations of functions used before they are defined */
109 /* */
110 /****************************************************************************/
111
112 /****************************************************************************/
113
114
115
116 /****************************************************************************/
117 /*
118 Gather_ElemDest -
119
120 SYNOPSIS:
121 static int Gather_ElemDest (DDD_OBJ obj, void *data);
122
123 PARAMETERS:
124 . obj
125 . data
126
127 DESCRIPTION:
128
129 RETURN VALUE:
130 int
131 */
132 /****************************************************************************/
133
134 static int Gather_ElemDest (DDD::DDDContext&, DDD_OBJ obj, void *data)
135 {
136 ELEMENT *theElement = (ELEMENT *)obj;
137
138 *(DDD_PROC *)data = PARTITION(theElement);
139
140 return 0;
141 }
142
143
144 /****************************************************************************/
145 /*
146 Scatter_ElemDest -
147
148 SYNOPSIS:
149 static int Scatter_ElemDest (DDD_OBJ obj, void *data);
150
151 PARAMETERS:
152 . obj
153 . data
154
155 DESCRIPTION:
156
157 RETURN VALUE:
158 int
159 */
160 /****************************************************************************/
161
162 static int Scatter_ElemDest (DDD::DDDContext&, DDD_OBJ obj, void *data)
163 {
164 ELEMENT *theElement = (ELEMENT *)obj;
165
166 PARTITION(theElement) = *(DDD_PROC *)data;
167
168 return 0;
169 }
170
171
172 /****************************************************************************/
173 /*
174 UpdateGhostDests -
175
176 SYNOPSIS:
177 static int UpdateGhostDests (MULTIGRID *theMG);
178
179 PARAMETERS:
180 . theMG
181
182 DESCRIPTION:
183
184 RETURN VALUE:
185 int
186 */
187 /****************************************************************************/
188
189 static int UpdateGhostDests (MULTIGRID *theMG)
190 {
191 auto& context = theMG->dddContext();
192 const auto& dddctrl = ddd_ctrl(context);
193
194 DDD_IFOneway(context,
195 dddctrl.ElementIF, IF_FORWARD, sizeof(DDD_PROC),
196 Gather_ElemDest, Scatter_ElemDest);
197
198 DDD_IFOneway(context,
199 dddctrl.ElementVIF, IF_FORWARD, sizeof(DDD_PROC),
200 Gather_ElemDest, Scatter_ElemDest);
201
202 return 0;
203 }
204
205
206
207 /****************************************************************************/
208
209
210 /****************************************************************************/
211 /*
212 Gather_GhostCmd -
213
214 SYNOPSIS:
215 static int Gather_GhostCmd (DDD_OBJ obj, void *data, DDD_PROC proc, DDD_PRIO prio);
216
217 PARAMETERS:
218 . obj
219 . data
220 . proc
221 . prio
222
223 DESCRIPTION:
224
225 RETURN VALUE:
226 int
227 */
228 /****************************************************************************/
229
230 static int Gather_GhostCmd (DDD::DDDContext&, DDD_OBJ obj, void *data, DDD_PROC proc, DDD_PRIO prio)
231 {
232 ELEMENT *theElement = (ELEMENT *)obj;
233 INT j;
234
235 /* TODO: not needed anymore. kb 9070108 */
236 if (PARTITION(theElement) == proc)
237 {
238 *((int *)data) = GC_ToMaster;
239
240 return(0);
241 }
242
243 if (PARTITION(theElement) != proc)
244 {
245 *((int *)data) = GC_Delete;
246
247 for(j=0; j<SIDES_OF_ELEM(theElement); j++)
248 {
249 ELEMENT *nb = NBELEM(theElement,j);
250
251 if (nb!=NULL)
252 {
253 if (PARTITION(nb)==proc)
254 {
255 *((int *)data) = GC_Keep;
256 break;
257 }
258 }
259 }
260 return(0);
261 }
262
263 return(1);
264 }
265
266
267 /****************************************************************************/
268 /*
269 Scatter_GhostCmd -
270
271 SYNOPSIS:
272 static int Scatter_GhostCmd (DDD_OBJ obj, void *data, DDD_PROC proc, DDD_PRIO prio);
273
274 PARAMETERS:
275 . obj
276 . data
277 . proc
278 . prio
279
280 DESCRIPTION:
281
282 RETURN VALUE:
283 int
284 */
285 /****************************************************************************/
286
287 static int Scatter_GhostCmd (DDD::DDDContext& context, DDD_OBJ obj, void *data, DDD_PROC proc, DDD_PRIO prio)
288 {
289 ELEMENT *theElement = (ELEMENT *)obj;
290 ELEMENT *SonList[MAX_SONS];
291 INT i;
292
293 const auto& me = context.me();
294
295 switch (*(int *)data)
296 {
297 case GC_Keep :
298 /* do nothing */
299 break;
300
301 case GC_ToMaster :
302 /* TODO: not needed anymore. kb 9070108
303 SETEPRIO(theElement, PrioMaster);*/
304 break;
305
306 case GC_Delete :
307 if (NSONS(theElement) > 0)
308 {
309 if (GetAllSons(theElement,SonList) != 0) return(1);
310 i = 0;
311 while (i<MAX_SONS && SonList[i] != NULL)
312 {
313 if (PARTITION(SonList[i]) == me) return(0);
314 i++;
315 }
316 }
317 XFEREDELETE(context, theElement);
318 break;
319
320 default :
321 assert(1);
322 }
323
324 return(0);
325 }
326
327
328 /****************************************************************************/
329 /*
330 Gather_VHGhostCmd -
331
332 SYNOPSIS:
333 static int Gather_VHGhostCmd (DDD_OBJ obj, void *data, DDD_PROC proc, DDD_PRIO prio);
334
335 PARAMETERS:
336 . obj
337 . data
338 . proc
339 . prio
340
341 DESCRIPTION:
342
343 RETURN VALUE:
344 int
345 */
346 /****************************************************************************/
347
348 static int Gather_VHGhostCmd (DDD::DDDContext&, DDD_OBJ obj, void *data, DDD_PROC proc, DDD_PRIO prio)
349 {
350 ELEMENT *theElement = (ELEMENT *)obj;
351 // ELEMENT *theFather = EFATHER(theElement);
352 INT j;
353
354 if (PARTITION(theElement) != proc)
355 {
356 *((int *)data) = GC_Delete;
357
358 for(j=0; j<SIDES_OF_ELEM(theElement); j++)
359 {
360 ELEMENT *theNeighbor = NBELEM(theElement,j);
361
362 if (theNeighbor != NULL)
363 {
364 if (PARTITION(theNeighbor) == proc)
365 {
366 *((int *)data) = GC_Keep;
367 return (0);
368 }
369 }
370 }
371
372 /* wrong: if (LEVEL(theElement) > 0)
373 {
374 ASSERT(theFather != NULL);
375
376 if (PARTITION(theFather) == proc)
377 {
378 *((int *)data) = GC_Keep;
379 return (0);
380 }
381 } */
382 return (0);
383 }
384
385 return(1);
386 }
387
388
389 /****************************************************************************/
390 /*
391 Scatter_VHGhostCmd -
392
393 SYNOPSIS:
394 static int Scatter_VHGhostCmd (DDD_OBJ obj, void *data, DDD_PROC proc, DDD_PRIO prio);
395
396 PARAMETERS:
397 . obj
398 . data
399 . proc
400 . prio
401
402 DESCRIPTION:
403
404 RETURN VALUE:
405 int
406 */
407 /****************************************************************************/
408
409 static int Scatter_VHGhostCmd (DDD::DDDContext& context, DDD_OBJ obj, void *data, DDD_PROC proc, DDD_PRIO prio)
410 {
411 const auto& me = context.me();
412
413 ELEMENT *theElement = (ELEMENT *)obj;
414 ELEMENT *SonList[MAX_SONS];
415
416 /* if element is needed after transfer here */
417 if ((*(int *)data) == GC_Keep) return(0);
418
419 /* element becomes master copy */
420 if (PARTITION(theElement) == me) return(0);
421
422 /* if a son resides as master keep element as vghost */
423 if (GetAllSons(theElement,SonList) != GM_OK) return(0);
424 INT i = 0;
425 while (i<MAX_SONS && SonList[i] != NULL)
426 {
427 if (PARTITION(SonList[i]) == me) return(0);
428 i++;
429 }
430 /* element is not needed on me any more */
431 if ((*(int *)data) == GC_Delete)
432 {
433 XFEREDELETE(context, theElement);
434 return(0);
435 }
436
437 return(1);
438 }
439
440
441 /****************************************************************************/
442 /*
443 ComputeGhostCmds -
444
445 SYNOPSIS:
446 static int ComputeGhostCmds (MULTIGRID *theMG);
447
448 PARAMETERS:
449 . theMG
450
451 DESCRIPTION:
452
453 RETURN VALUE:
454 int
455 */
456 /****************************************************************************/
457
458 static int ComputeGhostCmds (MULTIGRID *theMG)
459 {
460 auto& context = theMG->dddContext();
461 const auto& dddctrl = ddd_ctrl(context);
462
463 DDD_IFOnewayX(context,
464 dddctrl.ElementVHIF, IF_FORWARD, sizeof(int),
465 Gather_VHGhostCmd, Scatter_VHGhostCmd);
466
467 return(0);
468 }
469
470 /****************************************************************************/
471 /*
472 XferGridWithOverlap - send elements to other procs, keep overlapping region of one element, maintain correct priorities at interfaces.
473
474 SYNOPSIS:
475 static void XferGridWithOverlap (GRID *theGrid);
476
477 PARAMETERS:
478 . theGrid
479
480 DESCRIPTION:
481 This function sends elements to other procs, keeps overlapping region of one element and maintains correct priorities at interfaces. The destination procs have been computed by the RecursiveCoordinateBisection function and put into the elements' PARTITION-entries.
482
483 RETURN VALUE:
484 void
485 */
486 /****************************************************************************/
487
488 static int XferGridWithOverlap (GRID *theGrid)
489 {
490 ELEMENT *theElement, *theFather, *theNeighbor;
491 ELEMENT *SonList[MAX_SONS];
492 INT i,j,overlap_elem,part;
493 INT migrated = 0;
494
495 DDD::DDDContext& context = theGrid->dddContext();
496 const auto& me = context.me();
497
498 for(theElement=FIRSTELEMENT(theGrid); theElement!=NULL; theElement=SUCCE(theElement))
499 {
500 /* goal processor */
501 part = PARTITION(theElement);
502
503 /* create Master copy */
504 XferElement(context, theElement, part, PrioMaster);
505
506 #ifdef STAT_OUT
507 /* count elems */
508 if (part != me) migrated++;
509 #endif
510 }
511
512 /* create grid overlap */
513 for(theElement=FIRSTELEMENT(theGrid); theElement!=NULL; theElement=SUCCE(theElement))
514 {
515 overlap_elem = 0;
516
517 /* create 1-overlapping of horizontal elements */
518 for(j=0; j<SIDES_OF_ELEM(theElement); j++)
519 {
520 theNeighbor = NBELEM(theElement,j);
521
522 if (theNeighbor != NULL)
523 {
524 if (PARTITION(theElement)!=PARTITION(theNeighbor))
525 {
526 /* create Ghost copy */
527 XferElement(context, theElement, PARTITION(theNeighbor), PrioHGhost);
528 }
529
530 /* remember any local neighbour element */
531 if (PARTITION(theNeighbor)==me)
532 overlap_elem = 1;
533 }
534 }
535
536 /* create 1-overlapping of vertical elements */
537 theFather = EFATHER(theElement);
538 if (theFather != NULL)
539 {
540 if (PARTITION(theFather) != PARTITION(theElement) ||
541 !EMASTER(theFather))
542 /* create VGhost copy */
543 XferElement(context, theFather, PARTITION(theElement), PrioVGhost);
544 }
545 else
546 {
547 ASSERT(LEVEL(theElement) == 0);
548 }
549
550 /* consider elements on master-proc */
551 if (PARTITION(theElement)!=me)
552 {
553 if (NSONS(theElement) > 0)
554 {
555 if (GetAllSons(theElement,SonList) != 0) assert(0);
556 i = 0;
557 while (i<MAX_SONS && SonList[i] != NULL)
558 {
559 if (PARTITION(SonList[i]) == me)
560 {
561 overlap_elem += 2;
562 break;
563 }
564 i++;
565 }
566 }
567
568 PRINTDEBUG(dddif,1,("%d: XferGridWithOverlap(): elem=" EID_FMTX " p=%d new prio=%d\n",
569 me,EGID(theElement),PARTITION(theElement),overlap_elem));
570
571 if (overlap_elem > 0)
572 {
573 /* element is needed, set new prio */
574 switch (overlap_elem)
575 {
576 case (1) :
577 SETEPRIO(context, theElement,PrioHGhost);
578 break;
579
580 case (2) :
581 SETEPRIO(context, theElement,PrioVGhost);
582 break;
583
584 case (3) :
585 SETEPRIO(context, theElement,PrioVGhost);
586 break;
587
588 default :
589 assert(0);
590 }
591 }
592 else
593 {
594 /* element isn't needed */
595 PRINTDEBUG(dddif,2,("%d: XferGridWithOverlap(): XferDel elem=%d to p=%d prio=%d\n",
596 me,EGID(theElement),PARTITION(theElement),PrioHGhost));
597
598 XFEREDELETE(context, theElement);
599 }
600 }
601 }
602
603 return(migrated);
604 }
605
606
607
608 /****************************************************************************/
609
610
611 /****************************************************************************/
612 /*
613 InheritPartitionBottomTop -
614
615 SYNOPSIS:
616 static void InheritPartitionBottomTop (ELEMENT *e);
617
618 PARAMETERS:
619 . e
620
621 DESCRIPTION:
622
623 RETURN VALUE:
624 void
625 */
626 /****************************************************************************/
627
628 static void InheritPartitionBottomTop (ELEMENT *e)
629 {
630 int i;
631 ELEMENT *SonList[MAX_SONS];
632
633 if (GetSons(e,SonList) != GM_OK) assert(0);
634
635 for(i=0; i<MAX_SONS; i++)
636 {
637 ELEMENT *son = SonList[i];
638 if (son==NULL) break;
639
640 PARTITION(son) = PARTITION(e);
641 InheritPartitionBottomTop(son);
642 }
643 }
644
645
646 /****************************************************************************/
647
648
649 /****************************************************************************/
650 /*
651 TransferGridFromLevel -
652
653 SYNOPSIS:
654 int TransferGridFromLevel (MULTIGRID *theMG, INT level);
655
656 PARAMETERS:
657 . theMG
658 . level
659
660 DESCRIPTION:
661
662 RETURN VALUE:
663 int
664 */
665 /****************************************************************************/
666
667 int NS_DIM_PREFIX TransferGridFromLevel (MULTIGRID *theMG, INT level)
668 {
669 #ifdef STAT_OUT
670 INT migrated = 0; /* number of elements moved */
671 DOUBLE trans_begin, trans_end, cons_end;
672 #endif
673
674 #ifdef STAT_OUT
675 trans_begin = CURRENT_TIME;
676 #endif
677
678 /* send new destination to ghost elements */
679 UpdateGhostDests(theMG);
680
681 /* init transfer */
682 ddd_HandlerInit(theMG->dddContext(), HSET_XFER);
683
684 /* start physical transfer */
685 DDD_XferBegin(theMG->dddContext());
686
687 {
688 /* send 'commands' to ghosts in old partitioning */
689 ComputeGhostCmds(theMG);
690
691 /* send all grids */
692 for (INT g=0; g<=TOPLEVEL(theMG); g++)
693 {
694 GRID *theGrid = GRID_ON_LEVEL(theMG,g);
695 if (NT(theGrid)>0)
696 {
697 #ifdef STAT_OUT
698 migrated += XferGridWithOverlap(theGrid);
699 #else
700 XferGridWithOverlap(theGrid);
701 #endif
702 }
703 }
704 }
705
706 DDD_XferEnd(theMG->dddContext());
707
708 #ifdef STAT_OUT
709 trans_end = CURRENT_TIME;
710 #endif
711
712 /* set priorities of border nodes */
713 /* TODO this is an extra communication. eventually integrate this
714 with grid distribution phase. */
715 ConstructConsistentMultiGrid(theMG);
716
717 /* the grid has changed at least on one processor, thus reset MGSTATUS on all processors */
718 RESETMGSTATUS(theMG);
719
720 #ifdef STAT_OUT
721 cons_end = CURRENT_TIME;
722
723 /* sum up moved elements */
724 migrated = UG_GlobalSumINT(theMG->ppifContext(), migrated);
725
726 UserWriteF("MIGRATION: migrated=%d t_migrate=%.2f t_cons=%.2f\n",
727 migrated,trans_end-trans_begin,cons_end-trans_end);
728 #endif
729
730 #ifdef Debug
731 if (0)
732 DDD_ConsCheck(theMG->dddContext());
733 #endif
734
735 return 0;
736 }
737
738
739 /****************************************************************************/
740 /*
741 TransferGrid -
742
743 SYNOPSIS:
744 int TransferGrid (MULTIGRID *theMG);
745
746 PARAMETERS:
747 . theMG
748
749 DESCRIPTION:
750
751 RETURN VALUE:
752 int
753 */
754 /****************************************************************************/
755
756 int NS_DIM_PREFIX TransferGrid (MULTIGRID *theMG)
757 {
758 return TransferGridFromLevel(theMG,0);
759 }
760
761 /****************************************************************************/
762
763 #endif /* ModelP */