Debian Codegraph @

dune/uggrid/parallel/ddd/join/jcmds.cc

raw
1 // SPDX-FileCopyrightText: Copyright © DUNE Project contributors, see file LICENSE.md in module root
2 // SPDX-License-Identifier: LGPL-2.1-or-later
3 // -*- tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 2 -*-
4 // vi: set et ts=4 sw=2 sts=2:
5 /****************************************************************************/
6 /* */
7 /* File: jcmds.c */
8 /* */
9 /* Purpose: DDD-commands for Join Environment */
10 /* */
11 /* Author: Klaus Birken */
12 /* Institut fuer Computeranwendungen III */
13 /* Universitaet Stuttgart */
14 /* Pfaffenwaldring 27 */
15 /* 70569 Stuttgart */
16 /* email: [email protected] */
17 /* phone: 0049-(0)711-685-7007 */
18 /* fax : 0049-(0)711-685-7000 */
19 /* */
20 /* History: 980126 kb begin */
21 /* */
22 /* Remarks: */
23 /* */
24 /****************************************************************************/
25
26 /****************************************************************************/
27 /* */
28 /* include files */
29 /* system include files */
30 /* application include files */
31 /* */
32 /****************************************************************************/
33
34 /* standard C library */
35 #include <config.h>
36 #include <cstdlib>
37 #include <cstdio>
38 #include <cstring>
39
40 #include <algorithm>
41 #include <iomanip>
42
43 #include <dune/common/exceptions.hh>
44 #include <dune/common/stdstreams.hh>
45
46 #include <dune/uggrid/parallel/ddd/dddcontext.hh>
47
48 #include <dune/uggrid/parallel/ddd/dddi.h>
49 #include "join.h"
50
51
52 USING_UG_NAMESPACE
53 using namespace PPIF;
54
55 START_UGDIM_NAMESPACE
56
57 using namespace DDD::Join;
58
59 /****************************************************************************/
60 /* */
61 /* data structures */
62 /* */
63 /****************************************************************************/
64
65
66
67
68
69 /****************************************************************************/
70 /* */
71 /* definition of exported global variables */
72 /* */
73 /****************************************************************************/
74
75
76
77 /****************************************************************************/
78 /* */
79 /* definition of variables global to this source file only (static!) */
80 /* */
81 /****************************************************************************/
82
83
84
85
86 /****************************************************************************/
87 /* */
88 /* routines */
89 /* */
90 /****************************************************************************/
91
92
93 /*
94 prepare messages for phase 1.
95
96 */
97
98 static int PreparePhase1Msgs (DDD::DDDContext& context, std::vector<JIJoin*>& arrayJoin,
99 JOINMSG1 **theMsgs, size_t *memUsage)
100 {
101 auto& ctx = context.joinContext();
102
103 int i, last_i, nMsgs;
104 JIJoin** itemsJ = arrayJoin.data();
105 const int nJ = arrayJoin.size();
106
107 # if DebugJoin<=3
108 printf("%4d: PreparePhase1Msgs, nJoins=%d\n",
109 me, nJ);
110 fflush(stdout);
111 # endif
112
113 /* init return parameters */
114 *theMsgs = NULL;
115 *memUsage = 0;
116
117
118 if (nJ==0)
119 /* no messages */
120 return(0);
121
122
123 /* check whether Join objects are really local (without copies) */
124 /* and set local GID to invalid (will be set to new value lateron) */
125 for(i=0; i<nJ; i++)
126 {
127 if (ObjHasCpl(context, itemsJ[i]->hdr))
128 DUNE_THROW(Dune::Exception,
129 "cannot join " << OBJ_GID(itemsJ[i]->hdr)
130 << ", object already distributed");
131
132 OBJ_GID(itemsJ[i]->hdr) = GID_INVALID;
133 }
134
135
136 /* set local GID to new value */
137 for(i=0; i<nJ; i++)
138 {
139 DDD_GID local_gid = OBJ_GID(itemsJ[i]->hdr);
140
141 /* check for double Joins with different new_gid */
142 if (local_gid!=GID_INVALID && local_gid!=itemsJ[i]->new_gid)
143 DUNE_THROW(Dune::Exception,
144 "several (inconsistent) DDD_JoinObj-commands "
145 "for local object " << local_gid);
146
147 OBJ_GID(itemsJ[i]->hdr) = itemsJ[i]->new_gid;
148 }
149
150
151 nMsgs = 0;
152 last_i = i = 0;
153 do
154 {
155 size_t bufSize;
156
157 /* skip until dest-processor is different */
158 while (i<nJ && (itemsJ[i]->dest == itemsJ[last_i]->dest))
159 i++;
160
161 /* create new message */
162 JOINMSG1* jm = new JOINMSG1;
163 jm->nJoins = i-last_i;
164 jm->arrayJoin = &(itemsJ[last_i]);
165 jm->dest = itemsJ[last_i]->dest;
166 jm->next = *theMsgs;
167 *theMsgs = jm;
168 nMsgs++;
169
170 /* create new send message */
171 jm->msg_h = LC_NewSendMsg(context, ctx.phase1msg_t, jm->dest);
172
173 /* init table inside message */
174 LC_SetTableSize(jm->msg_h, ctx.jointab_id, jm->nJoins);
175
176 /* prepare message for sending away */
177 bufSize = LC_MsgPrepareSend(context, jm->msg_h);
178 *memUsage += bufSize;
179
180 if (DDD_GetOption(context, OPT_INFO_JOIN) & JOIN_SHOW_MEMUSAGE)
181 {
182 Dune::dwarn
183 << "DDD MESG [" << std::setw(3) << context.me() << "]: SHOW_MEM "
184 << "send msg phase1 dest=" << std::setw(4) << jm->dest
185 << " size=" << std::setw(10) << bufSize << "\n";
186 }
187
188 last_i = i;
189
190 } while (last_i < nJ);
191
192 return(nMsgs);
193 }
194
195
196
197 /****************************************************************************/
198 /* */
199 /* Function: PackPhase1Msgs */
200 /* */
201 /* Purpose: allocate one message buffer for each outgoing message, */
202 /* fill buffer with message contents and initiate asynchronous */
203 /* send for each message. */
204 /* */
205 /* Input: theMsgs: list of message-send-infos */
206 /* */
207 /* Output: - */
208 /* */
209 /****************************************************************************/
210
211 static void PackPhase1Msgs (DDD::DDDContext& context, JOINMSG1 *theMsgs)
212 {
213 auto& ctx = context.joinContext();
214 JOINMSG1 *jm;
215
216 for(jm=theMsgs; jm!=NULL; jm=jm->next)
217 {
218 TEJoin *theJoinTab;
219 int i;
220
221 /* copy data into message */
222 theJoinTab = (TEJoin *)LC_GetPtr(jm->msg_h, ctx.jointab_id);
223 for(i=0; i<jm->nJoins; i++)
224 {
225 theJoinTab[i].gid = jm->arrayJoin[i]->new_gid;
226 theJoinTab[i].prio = OBJ_PRIO(jm->arrayJoin[i]->hdr);
227 }
228 LC_SetTableLen(jm->msg_h, ctx.jointab_id, jm->nJoins);
229
230
231 /* send away */
232 LC_MsgSend(context, jm->msg_h);
233 }
234 }
235
236
237
238 /*
239 unpack phase1 messages.
240 */
241
242 static void UnpackPhase1Msgs (DDD::DDDContext& context,
243 LC_MSGHANDLE *theMsgs, int nRecvMsgs,
244 DDD_HDR *localCplObjs, int nLCO,
245 JIPartner **p_joinObjs, int *p_nJoinObjs)
246 {
247 auto& ctx = context.joinContext();
248 const auto& me = context.me();
249 JIPartner *joinObjs;
250 int nJoinObjs = 0;
251 int m, jo;
252
253 /* init return values */
254 *p_joinObjs = NULL;
255 *p_nJoinObjs = 0;
256
257
258 for(m=0; m<nRecvMsgs; m++)
259 {
260 LC_MSGHANDLE jm = theMsgs[m];
261 TEJoin *theJoin = (TEJoin *) LC_GetPtr(jm, ctx.jointab_id);
262 int nJ = (int) LC_GetTableLen(jm, ctx.jointab_id);
263 int i, j;
264
265 nJoinObjs += nJ;
266
267 for(i=0, j=0; i<nJ; i++)
268 {
269 while ((j<nLCO) && (OBJ_GID(localCplObjs[j]) < theJoin[i].gid))
270 j++;
271
272 if ((j<nLCO) && (OBJ_GID(localCplObjs[j])==theJoin[i].gid))
273 {
274 COUPLING *cpl;
275
276 /* found local object which is join target */
277 /* store shortcut to local object */
278 theJoin[i].hdr = localCplObjs[j];
279
280 /* generate phase2-JIAddCpl for this object */
281 for(cpl=ObjCplList(context, localCplObjs[j]); cpl!=NULL; cpl=CPL_NEXT(cpl))
282 {
283 JIAddCpl *ji = JIAddCplSet_NewItem(reinterpret_cast<JIAddCplSet*>(ctx.setJIAddCpl2));
284 ji->dest = CPL_PROC(cpl);
285 ji->te.gid = theJoin[i].gid;
286 ji->te.proc = LC_MsgGetProc(jm);
287 ji->te.prio = theJoin[i].prio;
288
289 if (! JIAddCplSet_ItemOK(reinterpret_cast<JIAddCplSet*>(ctx.setJIAddCpl2)))
290 continue;
291
292 # if DebugJoin<=1
293 printf("%4d: Phase1 Join for " DDD_GID_FMT " from %d, "
294 "send AddCpl to %d.\n",
295 me, theJoin[i].gid, ji->te.proc, ji->dest);
296 # endif
297
298 }
299
300 /* send phase3-JIAddCpl back to Join-proc */
301 for(cpl=ObjCplList(context, localCplObjs[j]); cpl!=NULL; cpl=CPL_NEXT(cpl))
302 {
303 JIAddCpl *ji = JIAddCplSet_NewItem(reinterpret_cast<JIAddCplSet*>(ctx.setJIAddCpl3));
304 ji->dest = LC_MsgGetProc(jm);
305 ji->te.gid = OBJ_GID(localCplObjs[j]);
306 ji->te.proc = CPL_PROC(cpl);
307 ji->te.prio = cpl->prio;
308
309 if (! JIAddCplSet_ItemOK(reinterpret_cast<JIAddCplSet*>(ctx.setJIAddCpl3)))
310 continue;
311 }
312 }
313 else
314 {
315 DUNE_THROW(Dune::Exception,
316 "no object " << theJoin[i].gid
317 << " for join from " << LC_MsgGetProc(jm));
318 }
319 }
320 }
321
322
323 /* return immediately if no join-objects have been found */
324 if (nJoinObjs==0)
325 return;
326
327
328 /* allocate array of objects, which has been contacted by a join */
329 joinObjs = new JIPartner[nJoinObjs];
330
331 /* set return values */
332 *p_joinObjs = joinObjs;
333 *p_nJoinObjs = nJoinObjs;
334
335
336 /* add one local coupling for each Join */
337 for(m=0, jo=0; m<nRecvMsgs; m++)
338 {
339 LC_MSGHANDLE jm = theMsgs[m];
340 TEJoin *theJoin = (TEJoin *) LC_GetPtr(jm, ctx.jointab_id);
341 int nJ = (int) LC_GetTableLen(jm, ctx.jointab_id);
342 int i;
343
344 for(i=0; i<nJ; i++)
345 {
346 AddCoupling(context, theJoin[i].hdr, LC_MsgGetProc(jm), theJoin[i].prio);
347
348 /* send one phase3-JIAddCpl for symmetric connection */
349 {
350 JIAddCpl *ji = JIAddCplSet_NewItem(reinterpret_cast<JIAddCplSet*>(ctx.setJIAddCpl3));
351 ji->dest = LC_MsgGetProc(jm);
352 ji->te.gid = OBJ_GID(theJoin[i].hdr);
353 ji->te.proc = me;
354 ji->te.prio = OBJ_PRIO(theJoin[i].hdr);
355
356 JIAddCplSet_ItemOK(reinterpret_cast<JIAddCplSet*>(ctx.setJIAddCpl3));
357 }
358
359 joinObjs[jo].hdr = theJoin[i].hdr;
360 joinObjs[jo].proc = LC_MsgGetProc(jm);
361 jo++;
362 }
363 }
364
365
366 /* sort joinObjs-array according to gid */
367 if (nJoinObjs>1) {
368 std::sort(
369 joinObjs, joinObjs + nJoinObjs,
370 [](const JIPartner& a, const JIPartner& b) {
371 return OBJ_GID(a.hdr) < OBJ_GID(b.hdr);
372 });
373 }
374 }
375
376
377
378 /****************************************************************************/
379
380 /*
381 prepare messages for phase 2.
382
383 */
384
385 static int PreparePhase2Msgs (DDD::DDDContext& context, std::vector<JIAddCpl*>& arrayAddCpl,
386 JOINMSG2 **theMsgs, size_t *memUsage)
387 {
388 auto& ctx = context.joinContext();
389 const auto& me = context.me();
390
391 int i, last_i, nMsgs;
392 JIAddCpl** itemsAC = arrayAddCpl.data();
393 const int nAC = arrayAddCpl.size();
394
395 # if DebugJoin<=3
396 printf("%4d: PreparePhase2Msgs, nAddCpls=%d\n",
397 me, nAC);
398 fflush(stdout);
399 # endif
400
401 /* init return parameters */
402 *theMsgs = NULL;
403 *memUsage = 0;
404
405
406 if (nAC==0)
407 /* no messages */
408 return(0);
409
410
411 nMsgs = 0;
412 last_i = i = 0;
413 do
414 {
415 size_t bufSize;
416
417 /* skip until dest-processor is different */
418 while (i<nAC && (itemsAC[i]->dest == itemsAC[last_i]->dest))
419 i++;
420
421 /* create new message */
422 JOINMSG2* jm = new JOINMSG2;
423 jm->nAddCpls = i-last_i;
424 jm->arrayAddCpl = &(itemsAC[last_i]);
425 jm->dest = itemsAC[last_i]->dest;
426 jm->next = *theMsgs;
427 *theMsgs = jm;
428 nMsgs++;
429
430 /* create new send message */
431 jm->msg_h = LC_NewSendMsg(context, ctx.phase2msg_t, jm->dest);
432
433 /* init table inside message */
434 LC_SetTableSize(jm->msg_h, ctx.addtab_id, jm->nAddCpls);
435
436 /* prepare message for sending away */
437 bufSize = LC_MsgPrepareSend(context, jm->msg_h);
438 *memUsage += bufSize;
439
440 if (DDD_GetOption(context, OPT_INFO_JOIN) & JOIN_SHOW_MEMUSAGE)
441 {
442 Dune::dwarn
443 << "DDD MESG [" << std::setw(3) << me << "]: SHOW_MEM "
444 << "send msg phase2 dest=" << std::setw(4) << jm->dest
445 << " size=" << std::setw(10) << bufSize << "\n";
446 }
447
448 last_i = i;
449
450 } while (last_i < nAC);
451
452 return(nMsgs);
453 }
454
455
456
457 /****************************************************************************/
458 /* */
459 /* Function: PackPhase2Msgs */
460 /* */
461 /* Purpose: allocate one message buffer for each outgoing message, */
462 /* fill buffer with message contents and initiate asynchronous */
463 /* send for each message. */
464 /* */
465 /* Input: theMsgs: list of message-send-infos */
466 /* */
467 /* Output: - */
468 /* */
469 /****************************************************************************/
470
471 static void PackPhase2Msgs(DDD::DDDContext& context, JOINMSG2 *theMsgs)
472 {
473 auto& ctx = context.joinContext();
474
475 JOINMSG2 *jm;
476
477 for(jm=theMsgs; jm!=NULL; jm=jm->next)
478 {
479 TEAddCpl *theAddTab;
480 int i;
481
482 /* copy data into message */
483 theAddTab = (TEAddCpl *)LC_GetPtr(jm->msg_h, ctx.addtab_id);
484 for(i=0; i<jm->nAddCpls; i++)
485 {
486 /* copy complete TEAddCpl item */
487 theAddTab[i] = jm->arrayAddCpl[i]->te;
488 }
489 LC_SetTableLen(jm->msg_h, ctx.addtab_id, jm->nAddCpls);
490
491
492 /* send away */
493 LC_MsgSend(context, jm->msg_h);
494 }
495 }
496
497
498
499 /*
500 unpack phase2 messages.
501 */
502
503 static void UnpackPhase2Msgs (DDD::DDDContext& context,
504 LC_MSGHANDLE *theMsgs2, int nRecvMsgs2,
505 JIPartner *joinObjs, int nJoinObjs,
506 DDD_HDR *localCplObjs, int nLCO)
507 {
508 auto& ctx = context.joinContext();
509
510 int m;
511
512 for(m=0; m<nRecvMsgs2; m++)
513 {
514 LC_MSGHANDLE jm = theMsgs2[m];
515 TEAddCpl *theAC = (TEAddCpl *) LC_GetPtr(jm, ctx.addtab_id);
516 int nAC = (int) LC_GetTableLen(jm, ctx.addtab_id);
517 int i, j, jo;
518
519 for(i=0, j=0, jo=0; i<nAC; i++)
520 {
521 while ((j<nLCO) && (OBJ_GID(localCplObjs[j]) < theAC[i].gid))
522 j++;
523
524 while ((jo<nJoinObjs) && (OBJ_GID(joinObjs[jo].hdr) < theAC[i].gid))
525 jo++;
526
527 if ((j<nLCO) && (OBJ_GID(localCplObjs[j])==theAC[i].gid))
528 {
529 /* found local object which is AddCpl target */
530 AddCoupling(context, localCplObjs[j], theAC[i].proc, theAC[i].prio);
531
532 # if DebugJoin<=1
533 printf("%4d: Phase2 execute AddCpl(%08x,%d,%d) (from %d).\n",
534 context.me(),
535 theAC[i].gid, theAC[i].proc, theAC[i].prio,
536 LC_MsgGetProc(jm));
537 # endif
538
539 while ((jo<nJoinObjs) && (OBJ_GID(joinObjs[jo].hdr) == theAC[i].gid))
540 {
541 JIAddCpl *ji = JIAddCplSet_NewItem(reinterpret_cast<JIAddCplSet*>(ctx.setJIAddCpl3));
542 ji->dest = joinObjs[jo].proc;
543 ji->te.gid = theAC[i].gid;
544 ji->te.proc = theAC[i].proc;
545 ji->te.prio = theAC[i].prio;
546 JIAddCplSet_ItemOK(reinterpret_cast<JIAddCplSet*>(ctx.setJIAddCpl3));
547
548
549 # if DebugJoin<=1
550 printf("%4d: Phase2 forward AddCpl(%08x,%d,%d) to %d.\n",
551 context.me(),
552 theAC[i].gid, theAC[i].proc, theAC[i].prio,
553 ji->dest);
554 # endif
555
556 jo++;
557 }
558 }
559 else
560 {
561 /* this should never happen. AddCpl send from invalid proc. */
562 assert(0);
563 }
564 }
565 }
566 }
567
568
569
570 /****************************************************************************/
571
572
573
574 /*
575 prepare messages for phase 3.
576
577 */
578
579 static int PreparePhase3Msgs (DDD::DDDContext& context, std::vector<JIAddCpl*>& arrayAddCpl,
580 JOINMSG3 **theMsgs, size_t *memUsage)
581 {
582 auto& ctx = context.joinContext();
583 const auto& me = context.me();
584
585 int i, last_i, nMsgs;
586 JIAddCpl** itemsAC = arrayAddCpl.data();
587 const int nAC = arrayAddCpl.size();
588
589 # if DebugJoin<=3
590 printf("%4d: PreparePhase3Msgs, nAddCpls=%d\n",
591 me, nAC);
592 fflush(stdout);
593 # endif
594
595 /* init return parameters */
596 *theMsgs = NULL;
597 *memUsage = 0;
598
599
600 if (nAC==0)
601 /* no messages */
602 return(0);
603
604
605 nMsgs = 0;
606 last_i = i = 0;
607 do
608 {
609 size_t bufSize;
610
611 /* skip until dest-processor is different */
612 while (i<nAC && (itemsAC[i]->dest == itemsAC[last_i]->dest))
613 i++;
614
615 /* create new message */
616 JOINMSG3* jm = new JOINMSG3;
617 jm->nAddCpls = i-last_i;
618 jm->arrayAddCpl = &(itemsAC[last_i]);
619 jm->dest = itemsAC[last_i]->dest;
620 jm->next = *theMsgs;
621 *theMsgs = jm;
622 nMsgs++;
623
624 /* create new send message */
625 jm->msg_h = LC_NewSendMsg(context, ctx.phase3msg_t, jm->dest);
626
627 /* init table inside message */
628 LC_SetTableSize(jm->msg_h, ctx.cpltab_id, jm->nAddCpls);
629
630 /* prepare message for sending away */
631 bufSize = LC_MsgPrepareSend(context, jm->msg_h);
632 *memUsage += bufSize;
633
634 if (DDD_GetOption(context, OPT_INFO_JOIN) & JOIN_SHOW_MEMUSAGE)
635 {
636 Dune::dwarn
637 << "DDD MESG [" << std::setw(3) << me << "]: SHOW_MEM "
638 << "send msg phase3 dest=" << std::setw(4) << jm->dest
639 << " size=" << std::setw(10) << bufSize << "\n";
640 }
641
642 last_i = i;
643
644 } while (last_i < nAC);
645
646 return(nMsgs);
647 }
648
649
650
651 /****************************************************************************/
652 /* */
653 /* Function: PackPhase3Msgs */
654 /* */
655 /* Purpose: allocate one message buffer for each outgoing message, */
656 /* fill buffer with message contents and initiate asynchronous */
657 /* send for each message. */
658 /* */
659 /* Input: theMsgs: list of message-send-infos */
660 /* */
661 /* Output: - */
662 /* */
663 /****************************************************************************/
664
665 static void PackPhase3Msgs(DDD::DDDContext& context, JOINMSG3 *theMsgs)
666 {
667 auto& ctx = context.joinContext();
668
669 JOINMSG3 *jm;
670
671 for(jm=theMsgs; jm!=NULL; jm=jm->next)
672 {
673 TEAddCpl *theAddTab;
674 int i;
675
676 /* copy data into message */
677 theAddTab = (TEAddCpl *)LC_GetPtr(jm->msg_h, ctx.cpltab_id);
678 for(i=0; i<jm->nAddCpls; i++)
679 {
680 /* copy complete TEAddCpl item */
681 theAddTab[i] = jm->arrayAddCpl[i]->te;
682 }
683 LC_SetTableLen(jm->msg_h, ctx.cpltab_id, jm->nAddCpls);
684
685
686 /* send away */
687 LC_MsgSend(context, jm->msg_h);
688 }
689 }
690
691
692
693 /****************************************************************************/
694
695
696 /*
697 unpack phase3 messages.
698 */
699
700 static void UnpackPhase3Msgs (DDD::DDDContext& context,
701 LC_MSGHANDLE *theMsgs, int nRecvMsgs,
702 std::vector<JIJoin*>& arrayJoin)
703 {
704 auto& ctx = context.joinContext();
705
706 JIJoin** itemsJ = arrayJoin.data();
707 const int nJ = arrayJoin.size();
708 int m;
709
710
711 for(m=0; m<nRecvMsgs; m++)
712 {
713 LC_MSGHANDLE jm = theMsgs[m];
714 TEAddCpl *theAC = (TEAddCpl *) LC_GetPtr(jm, ctx.cpltab_id);
715 int nAC = (int) LC_GetTableLen(jm, ctx.cpltab_id);
716 int i, j;
717
718 for(i=0, j=0; i<nAC; i++)
719 {
720 while ((j<nJ) && (OBJ_GID(itemsJ[j]->hdr) < theAC[i].gid))
721 j++;
722
723 if ((j<nJ) && (OBJ_GID(itemsJ[j]->hdr) == theAC[i].gid))
724 {
725 /* found local object which is AddCpl target */
726 AddCoupling(context, itemsJ[j]->hdr, theAC[i].proc, theAC[i].prio);
727
728 # if DebugJoin<=1
729 printf("%4d: Phase3 execute AddCpl(%08x,%d,%d) (from %d).\n",
730 context.me(),
731 OBJ_GID(itemsJ[j]->hdr), theAC[i].proc, theAC[i].prio,
732 LC_MsgGetProc(jm));
733 # endif
734 }
735 else
736 {
737 /* this should never happen. AddCpl send for unknown obj. */
738 assert(0);
739 }
740 }
741 }
742 }
743
744
745
746
747 /****************************************************************************/
748 /* */
749 /* Function: DDD_JoinEnd */
750 /* */
751 /****************************************************************************/
752
753 /**
754 End of join phase.
755 This function starts the actual join process. After a call to
756 this function (on all processors) all {\bf Join}-commands since
757 the last call to \funk{JoinBegin} are executed. This involves
758 a set of local communications between the processors.
759 */
760
761 DDD_RET DDD_JoinEnd(DDD::DDDContext& context)
762 {
763 auto& ctx = context.joinContext();
764
765 int obsolete, nRecvMsgs1, nRecvMsgs2, nRecvMsgs3;
766 JOINMSG1 *sendMsgs1=NULL, *sm1=NULL;
767 JOINMSG2 *sendMsgs2=NULL, *sm2=NULL;
768 JOINMSG3 *sendMsgs3=NULL, *sm3=NULL;
769 LC_MSGHANDLE *recvMsgs1=NULL, *recvMsgs2=NULL, *recvMsgs3=NULL;
770 size_t sendMem=0, recvMem=0;
771 JIPartner *joinObjs = NULL;
772 int nJoinObjs;
773 const auto& nCpls = context.couplingContext().nCpls;
774
775
776
777 STAT_SET_MODULE(DDD_MODULE_JOIN);
778 STAT_ZEROALL;
779
780 /* step mode and check whether call to JoinEnd is valid */
781 if (!JoinStepMode(context, JoinMode::JMODE_CMDS))
782 DUNE_THROW(Dune::Exception, "DDD_JoinEnd() aborted");
783
784
785 /*
786 PREPARATION PHASE
787 */
788 /* get sorted array of JIJoin-items */
789 std::vector<JIJoin*> arrayJIJoin = JIJoinSet_GetArray(reinterpret_cast<JIJoinSet*>(ctx.setJIJoin));
790 obsolete = JIJoinSet_GetNDiscarded(reinterpret_cast<JIJoinSet*>(ctx.setJIJoin));
791
792
793 /*
794 COMMUNICATION PHASE 1
795 all processors, where JoinObj-commands have been issued,
796 send information about these commands to the target
797 processors together with the GID of the objects on the
798 target procs and the local priority.
799 */
800 STAT_RESET;
801 /* prepare msgs for JIJoin-items */
802 PreparePhase1Msgs(context, arrayJIJoin, &sendMsgs1, &sendMem);
803 /* DisplayMemResources(); */
804
805 /* init communication topology */
806 nRecvMsgs1 = LC_Connect(context, ctx.phase1msg_t);
807 STAT_TIMER(T_JOIN_PREP_MSGS);
808
809 STAT_RESET;
810 /* build phase1 msgs on sender side and start send */
811 PackPhase1Msgs(context, sendMsgs1);
812 STAT_TIMER(T_JOIN_PACK_SEND);
813
814
815 /*
816 now messages are in the net, use spare time
817 */
818 STAT_RESET;
819 /* get sorted list of local objects with couplings */
820 std::vector<DDD_HDR> localCplObjs = LocalCoupledObjectsList(context);
821
822 if (obsolete>0)
823 {
824 if (DDD_GetOption(context, OPT_INFO_JOIN) & JOIN_SHOW_OBSOLETE)
825 {
826 int all = JIJoinSet_GetNItems(reinterpret_cast<JIJoinSet*>(ctx.setJIJoin));
827
828 using std::setw;
829 Dune::dwarn
830 << "DDD MESG [" << setw(3) << context.me() << "]: " << setw(4) << obsolete
831 << " from " << setw(4) << all << " join-cmds obsolete.\n";
832 }
833 }
834 STAT_TIMER(T_JOIN);
835
836
837 /*
838 nothing more to do until incoming messages arrive
839 */
840
841 /* display information about send-messages on lowcomm-level */
842 if (DDD_GetOption(context, OPT_INFO_JOIN) & JOIN_SHOW_MSGSALL)
843 {
844 DDD_SyncAll(context);
845 if (context.isMaster())
846 Dune::dwarn << "DDD JOIN_SHOW_MSGSALL: Phase1Msg.Send\n";
847 LC_PrintSendMsgs(context);
848 }
849
850
851 /* wait for communication-completion (send AND receive) */
852 STAT_RESET;
853 recvMsgs1 = LC_Communicate(context);
854 STAT_TIMER(T_JOIN_WAIT_RECV);
855
856
857 /* display information about message buffer sizes */
858 if (DDD_GetOption(context, OPT_INFO_JOIN) & JOIN_SHOW_MEMUSAGE)
859 {
860 int k;
861
862 /* sum up sizes of receive mesg buffers */
863 for(k=0; k<nRecvMsgs1; k++)
864 {
865 recvMem += LC_GetBufferSize(recvMsgs1[k]);
866 }
867
868 using std::setw;
869 Dune::dwarn
870 << "DDD MESG [" << setw(3) << context.me() << "]: SHOW_MEM msgs "
871 << " send=" << setw(10) << sendMem
872 << " recv=" << setw(10) << recvMem
873 << " all=" << setw(10) << (sendMem+recvMem) << "\n";
874 }
875
876 /* display information about recv-messages on lowcomm-level */
877 if (DDD_GetOption(context, OPT_INFO_JOIN) & JOIN_SHOW_MSGSALL)
878 {
879 DDD_SyncAll(context);
880 if (context.isMaster())
881 Dune::dwarn << "DDD JOIN_SHOW_MSGSALL: Phase1Msg.Recv\n";
882 LC_PrintRecvMsgs(context);
883 }
884
885 /* unpack messages */
886 STAT_RESET;
887 UnpackPhase1Msgs(context, recvMsgs1, nRecvMsgs1, localCplObjs.data(), nCpls,
888 &joinObjs, &nJoinObjs);
889 LC_Cleanup(context);
890 STAT_TIMER(T_JOIN_UNPACK);
891
892
893
894
895
896 /*
897 COMMUNICATION PHASE 2
898 all processors which received notification of JoinObj-commands
899 during phase 1 send AddCpl-requests to all copies of DDD objects,
900 for which Joins had been issued remotely.
901 */
902 /* get sorted array of JIAddCpl-items */
903 std::vector<JIAddCpl*> arrayJIAddCpl2 = JIAddCplSet_GetArray(reinterpret_cast<JIAddCplSet*>(ctx.setJIAddCpl2));
904
905 STAT_RESET;
906 /* prepare msgs for JIAddCpl-items */
907 PreparePhase2Msgs(context, arrayJIAddCpl2, &sendMsgs2, &sendMem);
908 /* DisplayMemResources(); */
909
910 /* init communication topology */
911 nRecvMsgs2 = LC_Connect(context, ctx.phase2msg_t);
912 STAT_TIMER(T_JOIN_PREP_MSGS);
913
914 STAT_RESET;
915 /* build phase2 msgs on sender side and start send */
916 PackPhase2Msgs(context, sendMsgs2);
917 STAT_TIMER(T_JOIN_PACK_SEND);
918
919 /*
920 now messages are in the net, use spare time
921 */
922
923 /* reorder Join-commands according to new_gid */
924 /* this ordering is needed in UnpackPhase3 */
925 if (arrayJIJoin.size() > 1)
926 {
927 std::sort(
928 arrayJIJoin.begin(), arrayJIJoin.end(),
929 [](const JIJoin* a, const JIJoin* b) {
930 return a->new_gid < b->new_gid;
931 });
932 }
933
934
935 /*
936 nothing more to do until incoming messages arrive
937 */
938
939 /* display information about send-messages on lowcomm-level */
940 if (DDD_GetOption(context, OPT_INFO_JOIN) & JOIN_SHOW_MSGSALL)
941 {
942 DDD_SyncAll(context);
943 if (context.isMaster())
944 Dune::dwarn <<"DDD JOIN_SHOW_MSGSALL: Phase2Msg.Send\n";
945 LC_PrintSendMsgs(context);
946 }
947
948
949 /* wait for communication-completion (send AND receive) */
950 STAT_RESET;
951 recvMsgs2 = LC_Communicate(context);
952 STAT_TIMER(T_JOIN_WAIT_RECV);
953
954
955 /* display information about message buffer sizes */
956 if (DDD_GetOption(context, OPT_INFO_JOIN) & JOIN_SHOW_MEMUSAGE)
957 {
958 int k;
959
960 /* sum up sizes of receive mesg buffers */
961 for(k=0; k<nRecvMsgs2; k++)
962 {
963 recvMem += LC_GetBufferSize(recvMsgs2[k]);
964 }
965
966 using std::setw;
967 Dune::dwarn
968 << "DDD MESG [" << setw(3) << context.me() << "]: SHOW_MEM msgs "
969 << " send=" << setw(10) << sendMem
970 << " recv=" << setw(10) << recvMem
971 << " all=" << setw(10) << (sendMem+recvMem) << "\n";
972 }
973
974 /* display information about recv-messages on lowcomm-level */
975 if (DDD_GetOption(context, OPT_INFO_JOIN) & JOIN_SHOW_MSGSALL)
976 {
977 DDD_SyncAll(context);
978 if (context.isMaster())
979 Dune::dwarn << "DDD JOIN_SHOW_MSGSALL: Phase2Msg.Recv\n";
980 LC_PrintRecvMsgs(context);
981 }
982
983 /* unpack messages */
984 STAT_RESET;
985 UnpackPhase2Msgs(context, recvMsgs2, nRecvMsgs2, joinObjs, nJoinObjs,
986 localCplObjs.data(), nCpls);
987
988 LC_Cleanup(context);
989 STAT_TIMER(T_JOIN_UNPACK);
990
991 for(; sendMsgs2!=NULL; sendMsgs2=sm2)
992 {
993 sm2 = sendMsgs2->next;
994 delete sendMsgs2;
995 }
996
997
998
999
1000
1001
1002 /*
1003 COMMUNICATION PHASE 3
1004 all processors which received notification of JoinObj-commands
1005 during phase 1 send AddCpl-requests to the procs where the
1006 JoinObj-commands have been issued. One AddCpl-request is sent
1007 for each cpl in the local objects coupling list. One AddCpl-request
1008 is sent for each AddCpl-request received during phase 2.
1009 (i.e., two kinds of AddCpl-requests are send to the processors
1010 on which the JoinObj-commands have been issued.
1011 */
1012 /* get sorted array of JIAddCpl-items */
1013 std::vector<JIAddCpl*> arrayJIAddCpl3 = JIAddCplSet_GetArray(reinterpret_cast<JIAddCplSet*>(ctx.setJIAddCpl3));
1014
1015 STAT_RESET;
1016 /* prepare msgs for JIAddCpl-items */
1017 PreparePhase3Msgs(context, arrayJIAddCpl3, &sendMsgs3, &sendMem);
1018 /* DisplayMemResources(); */
1019
1020 /* init communication topology */
1021 nRecvMsgs3 = LC_Connect(context, ctx.phase3msg_t);
1022 STAT_TIMER(T_JOIN_PREP_MSGS);
1023
1024 STAT_RESET;
1025 /* build phase3 msgs on sender side and start send */
1026 PackPhase3Msgs(context, sendMsgs3);
1027 STAT_TIMER(T_JOIN_PACK_SEND);
1028
1029 /*
1030 now messages are in the net, use spare time
1031 */
1032 /* ... */
1033
1034 /*
1035 nothing more to do until incoming messages arrive
1036 */
1037
1038 /* display information about send-messages on lowcomm-level */
1039 if (DDD_GetOption(context, OPT_INFO_JOIN) & JOIN_SHOW_MSGSALL)
1040 {
1041 DDD_SyncAll(context);
1042 if (context.isMaster())
1043 Dune::dwarn << "DDD JOIN_SHOW_MSGSALL: Phase3Msg.Send\n";
1044 LC_PrintSendMsgs(context);
1045 }
1046
1047
1048 /* wait for communication-completion (send AND receive) */
1049 STAT_RESET;
1050 recvMsgs3 = LC_Communicate(context);
1051 STAT_TIMER(T_JOIN_WAIT_RECV);
1052
1053
1054 /* display information about message buffer sizes */
1055 if (DDD_GetOption(context, OPT_INFO_JOIN) & JOIN_SHOW_MEMUSAGE)
1056 {
1057 int k;
1058
1059 /* sum up sizes of receive mesg buffers */
1060 for(k=0; k<nRecvMsgs3; k++)
1061 {
1062 recvMem += LC_GetBufferSize(recvMsgs3[k]);
1063 }
1064
1065 using std::setw;
1066 Dune::dwarn
1067 << "DDD MESG [" << setw(3) << context.me() << "]: SHOW_MEM msgs "
1068 << " send=" << setw(10) << sendMem
1069 << " recv=" << setw(10) << recvMem
1070 << " all=" << setw(10) << (sendMem+recvMem) << "\n";
1071 }
1072
1073 /* display information about recv-messages on lowcomm-level */
1074 if (DDD_GetOption(context, OPT_INFO_JOIN) & JOIN_SHOW_MSGSALL)
1075 {
1076 DDD_SyncAll(context);
1077 if (context.isMaster())
1078 Dune::dwarn << "DDD JOIN_SHOW_MSGSALL: Phase3Msg.Recv\n";
1079 LC_PrintRecvMsgs(context);
1080 }
1081
1082 /* unpack messages */
1083 STAT_RESET;
1084 UnpackPhase3Msgs(context, recvMsgs3, nRecvMsgs3, arrayJIJoin);
1085 LC_Cleanup(context);
1086 STAT_TIMER(T_JOIN_UNPACK);
1087
1088 for(; sendMsgs3!=NULL; sendMsgs3=sm3)
1089 {
1090 sm3 = sendMsgs3->next;
1091 delete sendMsgs3;
1092 }
1093
1094
1095
1096
1097
1098
1099 /*
1100 free temporary storage
1101 */
1102 JIJoinSet_Reset(reinterpret_cast<JIJoinSet*>(ctx.setJIJoin));
1103
1104 JIAddCplSet_Reset(reinterpret_cast<JIAddCplSet*>(ctx.setJIAddCpl2));
1105
1106 JIAddCplSet_Reset(reinterpret_cast<JIAddCplSet*>(ctx.setJIAddCpl3));
1107
1108 if (joinObjs!=NULL)
1109 delete[] joinObjs;
1110
1111 for(; sendMsgs1!=NULL; sendMsgs1=sm1)
1112 {
1113 sm1 = sendMsgs1->next;
1114 delete sendMsgs1;
1115 }
1116
1117
1118
1119 # if DebugJoin<=4
1120 Dune::dverb << "JoinEnd, before IFAllFromScratch().\n";
1121 # endif
1122
1123 /* re-create all interfaces and step JMODE */
1124 STAT_RESET;
1125 IFAllFromScratch(context);
1126 STAT_TIMER(T_JOIN_BUILD_IF);
1127
1128
1129 JoinStepMode(context, JoinMode::JMODE_BUSY);
1130
1131 return(DDD_RET_OK);
1132 }
1133
1134
1135
1136
1137
1138 /****************************************************************************/
1139 /* */
1140 /* Function: DDD_JoinObj */
1141 /* */
1142 /****************************************************************************/
1143
1144 /**
1145 Join local object with a distributed object.
1146
1147 \todoTBC
1148
1149 @param hdr DDD local object which should be joined.
1150 */
1151
1152
1153
1154 void DDD_JoinObj(DDD::DDDContext& context, DDD_HDR hdr, DDD_PROC dest, DDD_GID new_gid)
1155 {
1156 auto& ctx = context.joinContext();
1157 const auto procs = context.procs();
1158
1159 if (!ddd_JoinActive(context))
1160 DUNE_THROW(Dune::Exception, "Missing DDD_JoinBegin()");
1161
1162 if (dest>=procs)
1163 DUNE_THROW(Dune::Exception,
1164 "cannot join " << OBJ_GID(hdr) << " with " << new_gid
1165 << " on processor " << dest << " (procs=" << procs << ")");
1166
1167 if (dest==context.me())
1168 DUNE_THROW(Dune::Exception,
1169 "cannot join " << OBJ_GID(hdr) << " with myself");
1170
1171 if (ObjHasCpl(context, hdr))
1172 DUNE_THROW(Dune::Exception,
1173 "cannot join " << OBJ_GID(hdr)
1174 << ", object already distributed");
1175
1176
1177
1178 JIJoin* ji = JIJoinSet_NewItem(reinterpret_cast<JIJoinSet*>(ctx.setJIJoin));
1179 ji->hdr = hdr;
1180 ji->dest = dest;
1181 ji->new_gid = new_gid;
1182
1183 if (! JIJoinSet_ItemOK(reinterpret_cast<JIJoinSet*>(ctx.setJIJoin)))
1184 return;
1185
1186 # if DebugJoin<=2
1187 Dune:dvverb << "DDD_JoinObj " << OBJ_GID(hdr)
1188 << ", dest=" << dest << ", new_gid=" << new_gid << "\n";
1189 # endif
1190 }
1191
1192
1193
1194
1195 /****************************************************************************/
1196 /* */
1197 /* Function: DDD_JoinBegin */
1198 /* */
1199 /****************************************************************************/
1200
1201 /**
1202 Starts join phase.
1203 A call to this function establishes a global join operation.
1204 It must be issued on all processors. After this call an arbitrary
1205 series of {\bf Join}-commands may be issued. The global transfer operation
1206 is carried out via a \funk{JoinEnd} call on each processor.
1207 */
1208
1209 void DDD_JoinBegin(DDD::DDDContext& context)
1210 {
1211 /* step mode and check whether call to JoinBegin is valid */
1212 if (!JoinStepMode(context, JoinMode::JMODE_IDLE))
1213 DUNE_THROW(Dune::Exception, "DDD_JoinBegin() aborted");
1214 }
1215
1216
1217
1218 END_UGDIM_NAMESPACE