| 1 |
// SPDX-FileCopyrightText: Copyright © DUNE Project contributors, see file LICENSE.md in module root |
| 2 |
// SPDX-License-Identifier: LGPL-2.1-or-later |
| 3 |
// -*- tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 2 -*- |
| 4 |
// vi: set et ts=4 sw=2 sts=2: |
| 5 |
/****************************************************************************/ |
| 6 |
/* */ |
| 7 |
/* File: jcmds.c */ |
| 8 |
/* */ |
| 9 |
/* Purpose: DDD-commands for Join Environment */ |
| 10 |
/* */ |
| 11 |
/* Author: Klaus Birken */ |
| 12 |
/* Institut fuer Computeranwendungen III */ |
| 13 |
/* Universitaet Stuttgart */ |
| 14 |
/* Pfaffenwaldring 27 */ |
| 15 |
/* 70569 Stuttgart */ |
| 16 |
/* email: [email protected] */ |
| 17 |
/* phone: 0049-(0)711-685-7007 */ |
| 18 |
/* fax : 0049-(0)711-685-7000 */ |
| 19 |
/* */ |
| 20 |
/* History: 980126 kb begin */ |
| 21 |
/* */ |
| 22 |
/* Remarks: */ |
| 23 |
/* */ |
| 24 |
/****************************************************************************/ |
| 25 |
|
| 26 |
/****************************************************************************/ |
| 27 |
/* */ |
| 28 |
/* include files */ |
| 29 |
/* system include files */ |
| 30 |
/* application include files */ |
| 31 |
/* */ |
| 32 |
/****************************************************************************/ |
| 33 |
|
| 34 |
/* standard C library */ |
| 35 |
#include <config.h> |
| 36 |
#include <cstdlib> |
| 37 |
#include <cstdio> |
| 38 |
#include <cstring> |
| 39 |
|
| 40 |
#include <algorithm> |
| 41 |
#include <iomanip> |
| 42 |
|
| 43 |
#include <dune/common/exceptions.hh> |
| 44 |
#include <dune/common/stdstreams.hh> |
| 45 |
|
| 46 |
#include <dune/uggrid/parallel/ddd/dddcontext.hh> |
| 47 |
|
| 48 |
#include <dune/uggrid/parallel/ddd/dddi.h> |
| 49 |
#include "join.h" |
| 50 |
|
| 51 |
|
| 52 |
USING_UG_NAMESPACE |
| 53 |
using namespace PPIF; |
| 54 |
|
| 55 |
START_UGDIM_NAMESPACE |
| 56 |
|
| 57 |
using namespace DDD::Join; |
| 58 |
|
| 59 |
/****************************************************************************/ |
| 60 |
/* */ |
| 61 |
/* data structures */ |
| 62 |
/* */ |
| 63 |
/****************************************************************************/ |
| 64 |
|
| 65 |
|
| 66 |
|
| 67 |
|
| 68 |
|
| 69 |
/****************************************************************************/ |
| 70 |
/* */ |
| 71 |
/* definition of exported global variables */ |
| 72 |
/* */ |
| 73 |
/****************************************************************************/ |
| 74 |
|
| 75 |
|
| 76 |
|
| 77 |
/****************************************************************************/ |
| 78 |
/* */ |
| 79 |
/* definition of variables global to this source file only (static!) */ |
| 80 |
/* */ |
| 81 |
/****************************************************************************/ |
| 82 |
|
| 83 |
|
| 84 |
|
| 85 |
|
| 86 |
/****************************************************************************/ |
| 87 |
/* */ |
| 88 |
/* routines */ |
| 89 |
/* */ |
| 90 |
/****************************************************************************/ |
| 91 |
|
| 92 |
|
| 93 |
/* |
| 94 |
prepare messages for phase 1. |
| 95 |
|
| 96 |
*/ |
| 97 |
|
| 98 |
static int PreparePhase1Msgs (DDD::DDDContext& context, std::vector<JIJoin*>& arrayJoin, |
| 99 |
JOINMSG1 **theMsgs, size_t *memUsage) |
| 100 |
{ |
| 101 |
auto& ctx = context.joinContext(); |
| 102 |
|
| 103 |
int i, last_i, nMsgs; |
| 104 |
JIJoin** itemsJ = arrayJoin.data(); |
| 105 |
const int nJ = arrayJoin.size(); |
| 106 |
|
| 107 |
# if DebugJoin<=3 |
| 108 |
printf("%4d: PreparePhase1Msgs, nJoins=%d\n", |
| 109 |
me, nJ); |
| 110 |
fflush(stdout); |
| 111 |
# endif |
| 112 |
|
| 113 |
/* init return parameters */ |
| 114 |
*theMsgs = NULL; |
| 115 |
*memUsage = 0; |
| 116 |
|
| 117 |
|
| 118 |
if (nJ==0) |
| 119 |
/* no messages */ |
| 120 |
return(0); |
| 121 |
|
| 122 |
|
| 123 |
/* check whether Join objects are really local (without copies) */ |
| 124 |
/* and set local GID to invalid (will be set to new value lateron) */ |
| 125 |
for(i=0; i<nJ; i++) |
| 126 |
{ |
| 127 |
if (ObjHasCpl(context, itemsJ[i]->hdr)) |
| 128 |
DUNE_THROW(Dune::Exception, |
| 129 |
"cannot join " << OBJ_GID(itemsJ[i]->hdr) |
| 130 |
<< ", object already distributed"); |
| 131 |
|
| 132 |
OBJ_GID(itemsJ[i]->hdr) = GID_INVALID; |
| 133 |
} |
| 134 |
|
| 135 |
|
| 136 |
/* set local GID to new value */ |
| 137 |
for(i=0; i<nJ; i++) |
| 138 |
{ |
| 139 |
DDD_GID local_gid = OBJ_GID(itemsJ[i]->hdr); |
| 140 |
|
| 141 |
/* check for double Joins with different new_gid */ |
| 142 |
if (local_gid!=GID_INVALID && local_gid!=itemsJ[i]->new_gid) |
| 143 |
DUNE_THROW(Dune::Exception, |
| 144 |
"several (inconsistent) DDD_JoinObj-commands " |
| 145 |
"for local object " << local_gid); |
| 146 |
|
| 147 |
OBJ_GID(itemsJ[i]->hdr) = itemsJ[i]->new_gid; |
| 148 |
} |
| 149 |
|
| 150 |
|
| 151 |
nMsgs = 0; |
| 152 |
last_i = i = 0; |
| 153 |
do |
| 154 |
{ |
| 155 |
size_t bufSize; |
| 156 |
|
| 157 |
/* skip until dest-processor is different */ |
| 158 |
while (i<nJ && (itemsJ[i]->dest == itemsJ[last_i]->dest)) |
| 159 |
i++; |
| 160 |
|
| 161 |
/* create new message */ |
| 162 |
JOINMSG1* jm = new JOINMSG1; |
| 163 |
jm->nJoins = i-last_i; |
| 164 |
jm->arrayJoin = &(itemsJ[last_i]); |
| 165 |
jm->dest = itemsJ[last_i]->dest; |
| 166 |
jm->next = *theMsgs; |
| 167 |
*theMsgs = jm; |
| 168 |
nMsgs++; |
| 169 |
|
| 170 |
/* create new send message */ |
| 171 |
jm->msg_h = LC_NewSendMsg(context, ctx.phase1msg_t, jm->dest); |
| 172 |
|
| 173 |
/* init table inside message */ |
| 174 |
LC_SetTableSize(jm->msg_h, ctx.jointab_id, jm->nJoins); |
| 175 |
|
| 176 |
/* prepare message for sending away */ |
| 177 |
bufSize = LC_MsgPrepareSend(context, jm->msg_h); |
| 178 |
*memUsage += bufSize; |
| 179 |
|
| 180 |
if (DDD_GetOption(context, OPT_INFO_JOIN) & JOIN_SHOW_MEMUSAGE) |
| 181 |
{ |
| 182 |
Dune::dwarn |
| 183 |
<< "DDD MESG [" << std::setw(3) << context.me() << "]: SHOW_MEM " |
| 184 |
<< "send msg phase1 dest=" << std::setw(4) << jm->dest |
| 185 |
<< " size=" << std::setw(10) << bufSize << "\n"; |
| 186 |
} |
| 187 |
|
| 188 |
last_i = i; |
| 189 |
|
| 190 |
} while (last_i < nJ); |
| 191 |
|
| 192 |
return(nMsgs); |
| 193 |
} |
| 194 |
|
| 195 |
|
| 196 |
|
| 197 |
/****************************************************************************/ |
| 198 |
/* */ |
| 199 |
/* Function: PackPhase1Msgs */ |
| 200 |
/* */ |
| 201 |
/* Purpose: allocate one message buffer for each outgoing message, */ |
| 202 |
/* fill buffer with message contents and initiate asynchronous */ |
| 203 |
/* send for each message. */ |
| 204 |
/* */ |
| 205 |
/* Input: theMsgs: list of message-send-infos */ |
| 206 |
/* */ |
| 207 |
/* Output: - */ |
| 208 |
/* */ |
| 209 |
/****************************************************************************/ |
| 210 |
|
| 211 |
static void PackPhase1Msgs (DDD::DDDContext& context, JOINMSG1 *theMsgs) |
| 212 |
{ |
| 213 |
auto& ctx = context.joinContext(); |
| 214 |
JOINMSG1 *jm; |
| 215 |
|
| 216 |
for(jm=theMsgs; jm!=NULL; jm=jm->next) |
| 217 |
{ |
| 218 |
TEJoin *theJoinTab; |
| 219 |
int i; |
| 220 |
|
| 221 |
/* copy data into message */ |
| 222 |
theJoinTab = (TEJoin *)LC_GetPtr(jm->msg_h, ctx.jointab_id); |
| 223 |
for(i=0; i<jm->nJoins; i++) |
| 224 |
{ |
| 225 |
theJoinTab[i].gid = jm->arrayJoin[i]->new_gid; |
| 226 |
theJoinTab[i].prio = OBJ_PRIO(jm->arrayJoin[i]->hdr); |
| 227 |
} |
| 228 |
LC_SetTableLen(jm->msg_h, ctx.jointab_id, jm->nJoins); |
| 229 |
|
| 230 |
|
| 231 |
/* send away */ |
| 232 |
LC_MsgSend(context, jm->msg_h); |
| 233 |
} |
| 234 |
} |
| 235 |
|
| 236 |
|
| 237 |
|
| 238 |
/* |
| 239 |
unpack phase1 messages. |
| 240 |
*/ |
| 241 |
|
| 242 |
static void UnpackPhase1Msgs (DDD::DDDContext& context, |
| 243 |
LC_MSGHANDLE *theMsgs, int nRecvMsgs, |
| 244 |
DDD_HDR *localCplObjs, int nLCO, |
| 245 |
JIPartner **p_joinObjs, int *p_nJoinObjs) |
| 246 |
{ |
| 247 |
auto& ctx = context.joinContext(); |
| 248 |
const auto& me = context.me(); |
| 249 |
JIPartner *joinObjs; |
| 250 |
int nJoinObjs = 0; |
| 251 |
int m, jo; |
| 252 |
|
| 253 |
/* init return values */ |
| 254 |
*p_joinObjs = NULL; |
| 255 |
*p_nJoinObjs = 0; |
| 256 |
|
| 257 |
|
| 258 |
for(m=0; m<nRecvMsgs; m++) |
| 259 |
{ |
| 260 |
LC_MSGHANDLE jm = theMsgs[m]; |
| 261 |
TEJoin *theJoin = (TEJoin *) LC_GetPtr(jm, ctx.jointab_id); |
| 262 |
int nJ = (int) LC_GetTableLen(jm, ctx.jointab_id); |
| 263 |
int i, j; |
| 264 |
|
| 265 |
nJoinObjs += nJ; |
| 266 |
|
| 267 |
for(i=0, j=0; i<nJ; i++) |
| 268 |
{ |
| 269 |
while ((j<nLCO) && (OBJ_GID(localCplObjs[j]) < theJoin[i].gid)) |
| 270 |
j++; |
| 271 |
|
| 272 |
if ((j<nLCO) && (OBJ_GID(localCplObjs[j])==theJoin[i].gid)) |
| 273 |
{ |
| 274 |
COUPLING *cpl; |
| 275 |
|
| 276 |
/* found local object which is join target */ |
| 277 |
/* store shortcut to local object */ |
| 278 |
theJoin[i].hdr = localCplObjs[j]; |
| 279 |
|
| 280 |
/* generate phase2-JIAddCpl for this object */ |
| 281 |
for(cpl=ObjCplList(context, localCplObjs[j]); cpl!=NULL; cpl=CPL_NEXT(cpl)) |
| 282 |
{ |
| 283 |
JIAddCpl *ji = JIAddCplSet_NewItem(reinterpret_cast<JIAddCplSet*>(ctx.setJIAddCpl2)); |
| 284 |
ji->dest = CPL_PROC(cpl); |
| 285 |
ji->te.gid = theJoin[i].gid; |
| 286 |
ji->te.proc = LC_MsgGetProc(jm); |
| 287 |
ji->te.prio = theJoin[i].prio; |
| 288 |
|
| 289 |
if (! JIAddCplSet_ItemOK(reinterpret_cast<JIAddCplSet*>(ctx.setJIAddCpl2))) |
| 290 |
continue; |
| 291 |
|
| 292 |
# if DebugJoin<=1 |
| 293 |
printf("%4d: Phase1 Join for " DDD_GID_FMT " from %d, " |
| 294 |
"send AddCpl to %d.\n", |
| 295 |
me, theJoin[i].gid, ji->te.proc, ji->dest); |
| 296 |
# endif |
| 297 |
|
| 298 |
} |
| 299 |
|
| 300 |
/* send phase3-JIAddCpl back to Join-proc */ |
| 301 |
for(cpl=ObjCplList(context, localCplObjs[j]); cpl!=NULL; cpl=CPL_NEXT(cpl)) |
| 302 |
{ |
| 303 |
JIAddCpl *ji = JIAddCplSet_NewItem(reinterpret_cast<JIAddCplSet*>(ctx.setJIAddCpl3)); |
| 304 |
ji->dest = LC_MsgGetProc(jm); |
| 305 |
ji->te.gid = OBJ_GID(localCplObjs[j]); |
| 306 |
ji->te.proc = CPL_PROC(cpl); |
| 307 |
ji->te.prio = cpl->prio; |
| 308 |
|
| 309 |
if (! JIAddCplSet_ItemOK(reinterpret_cast<JIAddCplSet*>(ctx.setJIAddCpl3))) |
| 310 |
continue; |
| 311 |
} |
| 312 |
} |
| 313 |
else |
| 314 |
{ |
| 315 |
DUNE_THROW(Dune::Exception, |
| 316 |
"no object " << theJoin[i].gid |
| 317 |
<< " for join from " << LC_MsgGetProc(jm)); |
| 318 |
} |
| 319 |
} |
| 320 |
} |
| 321 |
|
| 322 |
|
| 323 |
/* return immediately if no join-objects have been found */ |
| 324 |
if (nJoinObjs==0) |
| 325 |
return; |
| 326 |
|
| 327 |
|
| 328 |
/* allocate array of objects, which has been contacted by a join */ |
| 329 |
joinObjs = new JIPartner[nJoinObjs]; |
| 330 |
|
| 331 |
/* set return values */ |
| 332 |
*p_joinObjs = joinObjs; |
| 333 |
*p_nJoinObjs = nJoinObjs; |
| 334 |
|
| 335 |
|
| 336 |
/* add one local coupling for each Join */ |
| 337 |
for(m=0, jo=0; m<nRecvMsgs; m++) |
| 338 |
{ |
| 339 |
LC_MSGHANDLE jm = theMsgs[m]; |
| 340 |
TEJoin *theJoin = (TEJoin *) LC_GetPtr(jm, ctx.jointab_id); |
| 341 |
int nJ = (int) LC_GetTableLen(jm, ctx.jointab_id); |
| 342 |
int i; |
| 343 |
|
| 344 |
for(i=0; i<nJ; i++) |
| 345 |
{ |
| 346 |
AddCoupling(context, theJoin[i].hdr, LC_MsgGetProc(jm), theJoin[i].prio); |
| 347 |
|
| 348 |
/* send one phase3-JIAddCpl for symmetric connection */ |
| 349 |
{ |
| 350 |
JIAddCpl *ji = JIAddCplSet_NewItem(reinterpret_cast<JIAddCplSet*>(ctx.setJIAddCpl3)); |
| 351 |
ji->dest = LC_MsgGetProc(jm); |
| 352 |
ji->te.gid = OBJ_GID(theJoin[i].hdr); |
| 353 |
ji->te.proc = me; |
| 354 |
ji->te.prio = OBJ_PRIO(theJoin[i].hdr); |
| 355 |
|
| 356 |
JIAddCplSet_ItemOK(reinterpret_cast<JIAddCplSet*>(ctx.setJIAddCpl3)); |
| 357 |
} |
| 358 |
|
| 359 |
joinObjs[jo].hdr = theJoin[i].hdr; |
| 360 |
joinObjs[jo].proc = LC_MsgGetProc(jm); |
| 361 |
jo++; |
| 362 |
} |
| 363 |
} |
| 364 |
|
| 365 |
|
| 366 |
/* sort joinObjs-array according to gid */ |
| 367 |
if (nJoinObjs>1) { |
| 368 |
std::sort( |
| 369 |
joinObjs, joinObjs + nJoinObjs, |
| 370 |
[](const JIPartner& a, const JIPartner& b) { |
| 371 |
return OBJ_GID(a.hdr) < OBJ_GID(b.hdr); |
| 372 |
}); |
| 373 |
} |
| 374 |
} |
| 375 |
|
| 376 |
|
| 377 |
|
| 378 |
/****************************************************************************/ |
| 379 |
|
| 380 |
/* |
| 381 |
prepare messages for phase 2. |
| 382 |
|
| 383 |
*/ |
| 384 |
|
| 385 |
static int PreparePhase2Msgs (DDD::DDDContext& context, std::vector<JIAddCpl*>& arrayAddCpl, |
| 386 |
JOINMSG2 **theMsgs, size_t *memUsage) |
| 387 |
{ |
| 388 |
auto& ctx = context.joinContext(); |
| 389 |
const auto& me = context.me(); |
| 390 |
|
| 391 |
int i, last_i, nMsgs; |
| 392 |
JIAddCpl** itemsAC = arrayAddCpl.data(); |
| 393 |
const int nAC = arrayAddCpl.size(); |
| 394 |
|
| 395 |
# if DebugJoin<=3 |
| 396 |
printf("%4d: PreparePhase2Msgs, nAddCpls=%d\n", |
| 397 |
me, nAC); |
| 398 |
fflush(stdout); |
| 399 |
# endif |
| 400 |
|
| 401 |
/* init return parameters */ |
| 402 |
*theMsgs = NULL; |
| 403 |
*memUsage = 0; |
| 404 |
|
| 405 |
|
| 406 |
if (nAC==0) |
| 407 |
/* no messages */ |
| 408 |
return(0); |
| 409 |
|
| 410 |
|
| 411 |
nMsgs = 0; |
| 412 |
last_i = i = 0; |
| 413 |
do |
| 414 |
{ |
| 415 |
size_t bufSize; |
| 416 |
|
| 417 |
/* skip until dest-processor is different */ |
| 418 |
while (i<nAC && (itemsAC[i]->dest == itemsAC[last_i]->dest)) |
| 419 |
i++; |
| 420 |
|
| 421 |
/* create new message */ |
| 422 |
JOINMSG2* jm = new JOINMSG2; |
| 423 |
jm->nAddCpls = i-last_i; |
| 424 |
jm->arrayAddCpl = &(itemsAC[last_i]); |
| 425 |
jm->dest = itemsAC[last_i]->dest; |
| 426 |
jm->next = *theMsgs; |
| 427 |
*theMsgs = jm; |
| 428 |
nMsgs++; |
| 429 |
|
| 430 |
/* create new send message */ |
| 431 |
jm->msg_h = LC_NewSendMsg(context, ctx.phase2msg_t, jm->dest); |
| 432 |
|
| 433 |
/* init table inside message */ |
| 434 |
LC_SetTableSize(jm->msg_h, ctx.addtab_id, jm->nAddCpls); |
| 435 |
|
| 436 |
/* prepare message for sending away */ |
| 437 |
bufSize = LC_MsgPrepareSend(context, jm->msg_h); |
| 438 |
*memUsage += bufSize; |
| 439 |
|
| 440 |
if (DDD_GetOption(context, OPT_INFO_JOIN) & JOIN_SHOW_MEMUSAGE) |
| 441 |
{ |
| 442 |
Dune::dwarn |
| 443 |
<< "DDD MESG [" << std::setw(3) << me << "]: SHOW_MEM " |
| 444 |
<< "send msg phase2 dest=" << std::setw(4) << jm->dest |
| 445 |
<< " size=" << std::setw(10) << bufSize << "\n"; |
| 446 |
} |
| 447 |
|
| 448 |
last_i = i; |
| 449 |
|
| 450 |
} while (last_i < nAC); |
| 451 |
|
| 452 |
return(nMsgs); |
| 453 |
} |
| 454 |
|
| 455 |
|
| 456 |
|
| 457 |
/****************************************************************************/ |
| 458 |
/* */ |
| 459 |
/* Function: PackPhase2Msgs */ |
| 460 |
/* */ |
| 461 |
/* Purpose: allocate one message buffer for each outgoing message, */ |
| 462 |
/* fill buffer with message contents and initiate asynchronous */ |
| 463 |
/* send for each message. */ |
| 464 |
/* */ |
| 465 |
/* Input: theMsgs: list of message-send-infos */ |
| 466 |
/* */ |
| 467 |
/* Output: - */ |
| 468 |
/* */ |
| 469 |
/****************************************************************************/ |
| 470 |
|
| 471 |
static void PackPhase2Msgs(DDD::DDDContext& context, JOINMSG2 *theMsgs) |
| 472 |
{ |
| 473 |
auto& ctx = context.joinContext(); |
| 474 |
|
| 475 |
JOINMSG2 *jm; |
| 476 |
|
| 477 |
for(jm=theMsgs; jm!=NULL; jm=jm->next) |
| 478 |
{ |
| 479 |
TEAddCpl *theAddTab; |
| 480 |
int i; |
| 481 |
|
| 482 |
/* copy data into message */ |
| 483 |
theAddTab = (TEAddCpl *)LC_GetPtr(jm->msg_h, ctx.addtab_id); |
| 484 |
for(i=0; i<jm->nAddCpls; i++) |
| 485 |
{ |
| 486 |
/* copy complete TEAddCpl item */ |
| 487 |
theAddTab[i] = jm->arrayAddCpl[i]->te; |
| 488 |
} |
| 489 |
LC_SetTableLen(jm->msg_h, ctx.addtab_id, jm->nAddCpls); |
| 490 |
|
| 491 |
|
| 492 |
/* send away */ |
| 493 |
LC_MsgSend(context, jm->msg_h); |
| 494 |
} |
| 495 |
} |
| 496 |
|
| 497 |
|
| 498 |
|
| 499 |
/* |
| 500 |
unpack phase2 messages. |
| 501 |
*/ |
| 502 |
|
| 503 |
static void UnpackPhase2Msgs (DDD::DDDContext& context, |
| 504 |
LC_MSGHANDLE *theMsgs2, int nRecvMsgs2, |
| 505 |
JIPartner *joinObjs, int nJoinObjs, |
| 506 |
DDD_HDR *localCplObjs, int nLCO) |
| 507 |
{ |
| 508 |
auto& ctx = context.joinContext(); |
| 509 |
|
| 510 |
int m; |
| 511 |
|
| 512 |
for(m=0; m<nRecvMsgs2; m++) |
| 513 |
{ |
| 514 |
LC_MSGHANDLE jm = theMsgs2[m]; |
| 515 |
TEAddCpl *theAC = (TEAddCpl *) LC_GetPtr(jm, ctx.addtab_id); |
| 516 |
int nAC = (int) LC_GetTableLen(jm, ctx.addtab_id); |
| 517 |
int i, j, jo; |
| 518 |
|
| 519 |
for(i=0, j=0, jo=0; i<nAC; i++) |
| 520 |
{ |
| 521 |
while ((j<nLCO) && (OBJ_GID(localCplObjs[j]) < theAC[i].gid)) |
| 522 |
j++; |
| 523 |
|
| 524 |
while ((jo<nJoinObjs) && (OBJ_GID(joinObjs[jo].hdr) < theAC[i].gid)) |
| 525 |
jo++; |
| 526 |
|
| 527 |
if ((j<nLCO) && (OBJ_GID(localCplObjs[j])==theAC[i].gid)) |
| 528 |
{ |
| 529 |
/* found local object which is AddCpl target */ |
| 530 |
AddCoupling(context, localCplObjs[j], theAC[i].proc, theAC[i].prio); |
| 531 |
|
| 532 |
# if DebugJoin<=1 |
| 533 |
printf("%4d: Phase2 execute AddCpl(%08x,%d,%d) (from %d).\n", |
| 534 |
context.me(), |
| 535 |
theAC[i].gid, theAC[i].proc, theAC[i].prio, |
| 536 |
LC_MsgGetProc(jm)); |
| 537 |
# endif |
| 538 |
|
| 539 |
while ((jo<nJoinObjs) && (OBJ_GID(joinObjs[jo].hdr) == theAC[i].gid)) |
| 540 |
{ |
| 541 |
JIAddCpl *ji = JIAddCplSet_NewItem(reinterpret_cast<JIAddCplSet*>(ctx.setJIAddCpl3)); |
| 542 |
ji->dest = joinObjs[jo].proc; |
| 543 |
ji->te.gid = theAC[i].gid; |
| 544 |
ji->te.proc = theAC[i].proc; |
| 545 |
ji->te.prio = theAC[i].prio; |
| 546 |
JIAddCplSet_ItemOK(reinterpret_cast<JIAddCplSet*>(ctx.setJIAddCpl3)); |
| 547 |
|
| 548 |
|
| 549 |
# if DebugJoin<=1 |
| 550 |
printf("%4d: Phase2 forward AddCpl(%08x,%d,%d) to %d.\n", |
| 551 |
context.me(), |
| 552 |
theAC[i].gid, theAC[i].proc, theAC[i].prio, |
| 553 |
ji->dest); |
| 554 |
# endif |
| 555 |
|
| 556 |
jo++; |
| 557 |
} |
| 558 |
} |
| 559 |
else |
| 560 |
{ |
| 561 |
/* this should never happen. AddCpl send from invalid proc. */ |
| 562 |
assert(0); |
| 563 |
} |
| 564 |
} |
| 565 |
} |
| 566 |
} |
| 567 |
|
| 568 |
|
| 569 |
|
| 570 |
/****************************************************************************/ |
| 571 |
|
| 572 |
|
| 573 |
|
| 574 |
/* |
| 575 |
prepare messages for phase 3. |
| 576 |
|
| 577 |
*/ |
| 578 |
|
| 579 |
static int PreparePhase3Msgs (DDD::DDDContext& context, std::vector<JIAddCpl*>& arrayAddCpl, |
| 580 |
JOINMSG3 **theMsgs, size_t *memUsage) |
| 581 |
{ |
| 582 |
auto& ctx = context.joinContext(); |
| 583 |
const auto& me = context.me(); |
| 584 |
|
| 585 |
int i, last_i, nMsgs; |
| 586 |
JIAddCpl** itemsAC = arrayAddCpl.data(); |
| 587 |
const int nAC = arrayAddCpl.size(); |
| 588 |
|
| 589 |
# if DebugJoin<=3 |
| 590 |
printf("%4d: PreparePhase3Msgs, nAddCpls=%d\n", |
| 591 |
me, nAC); |
| 592 |
fflush(stdout); |
| 593 |
# endif |
| 594 |
|
| 595 |
/* init return parameters */ |
| 596 |
*theMsgs = NULL; |
| 597 |
*memUsage = 0; |
| 598 |
|
| 599 |
|
| 600 |
if (nAC==0) |
| 601 |
/* no messages */ |
| 602 |
return(0); |
| 603 |
|
| 604 |
|
| 605 |
nMsgs = 0; |
| 606 |
last_i = i = 0; |
| 607 |
do |
| 608 |
{ |
| 609 |
size_t bufSize; |
| 610 |
|
| 611 |
/* skip until dest-processor is different */ |
| 612 |
while (i<nAC && (itemsAC[i]->dest == itemsAC[last_i]->dest)) |
| 613 |
i++; |
| 614 |
|
| 615 |
/* create new message */ |
| 616 |
JOINMSG3* jm = new JOINMSG3; |
| 617 |
jm->nAddCpls = i-last_i; |
| 618 |
jm->arrayAddCpl = &(itemsAC[last_i]); |
| 619 |
jm->dest = itemsAC[last_i]->dest; |
| 620 |
jm->next = *theMsgs; |
| 621 |
*theMsgs = jm; |
| 622 |
nMsgs++; |
| 623 |
|
| 624 |
/* create new send message */ |
| 625 |
jm->msg_h = LC_NewSendMsg(context, ctx.phase3msg_t, jm->dest); |
| 626 |
|
| 627 |
/* init table inside message */ |
| 628 |
LC_SetTableSize(jm->msg_h, ctx.cpltab_id, jm->nAddCpls); |
| 629 |
|
| 630 |
/* prepare message for sending away */ |
| 631 |
bufSize = LC_MsgPrepareSend(context, jm->msg_h); |
| 632 |
*memUsage += bufSize; |
| 633 |
|
| 634 |
if (DDD_GetOption(context, OPT_INFO_JOIN) & JOIN_SHOW_MEMUSAGE) |
| 635 |
{ |
| 636 |
Dune::dwarn |
| 637 |
<< "DDD MESG [" << std::setw(3) << me << "]: SHOW_MEM " |
| 638 |
<< "send msg phase3 dest=" << std::setw(4) << jm->dest |
| 639 |
<< " size=" << std::setw(10) << bufSize << "\n"; |
| 640 |
} |
| 641 |
|
| 642 |
last_i = i; |
| 643 |
|
| 644 |
} while (last_i < nAC); |
| 645 |
|
| 646 |
return(nMsgs); |
| 647 |
} |
| 648 |
|
| 649 |
|
| 650 |
|
| 651 |
/****************************************************************************/ |
| 652 |
/* */ |
| 653 |
/* Function: PackPhase3Msgs */ |
| 654 |
/* */ |
| 655 |
/* Purpose: allocate one message buffer for each outgoing message, */ |
| 656 |
/* fill buffer with message contents and initiate asynchronous */ |
| 657 |
/* send for each message. */ |
| 658 |
/* */ |
| 659 |
/* Input: theMsgs: list of message-send-infos */ |
| 660 |
/* */ |
| 661 |
/* Output: - */ |
| 662 |
/* */ |
| 663 |
/****************************************************************************/ |
| 664 |
|
| 665 |
static void PackPhase3Msgs(DDD::DDDContext& context, JOINMSG3 *theMsgs) |
| 666 |
{ |
| 667 |
auto& ctx = context.joinContext(); |
| 668 |
|
| 669 |
JOINMSG3 *jm; |
| 670 |
|
| 671 |
for(jm=theMsgs; jm!=NULL; jm=jm->next) |
| 672 |
{ |
| 673 |
TEAddCpl *theAddTab; |
| 674 |
int i; |
| 675 |
|
| 676 |
/* copy data into message */ |
| 677 |
theAddTab = (TEAddCpl *)LC_GetPtr(jm->msg_h, ctx.cpltab_id); |
| 678 |
for(i=0; i<jm->nAddCpls; i++) |
| 679 |
{ |
| 680 |
/* copy complete TEAddCpl item */ |
| 681 |
theAddTab[i] = jm->arrayAddCpl[i]->te; |
| 682 |
} |
| 683 |
LC_SetTableLen(jm->msg_h, ctx.cpltab_id, jm->nAddCpls); |
| 684 |
|
| 685 |
|
| 686 |
/* send away */ |
| 687 |
LC_MsgSend(context, jm->msg_h); |
| 688 |
} |
| 689 |
} |
| 690 |
|
| 691 |
|
| 692 |
|
| 693 |
/****************************************************************************/ |
| 694 |
|
| 695 |
|
| 696 |
/* |
| 697 |
unpack phase3 messages. |
| 698 |
*/ |
| 699 |
|
| 700 |
static void UnpackPhase3Msgs (DDD::DDDContext& context, |
| 701 |
LC_MSGHANDLE *theMsgs, int nRecvMsgs, |
| 702 |
std::vector<JIJoin*>& arrayJoin) |
| 703 |
{ |
| 704 |
auto& ctx = context.joinContext(); |
| 705 |
|
| 706 |
JIJoin** itemsJ = arrayJoin.data(); |
| 707 |
const int nJ = arrayJoin.size(); |
| 708 |
int m; |
| 709 |
|
| 710 |
|
| 711 |
for(m=0; m<nRecvMsgs; m++) |
| 712 |
{ |
| 713 |
LC_MSGHANDLE jm = theMsgs[m]; |
| 714 |
TEAddCpl *theAC = (TEAddCpl *) LC_GetPtr(jm, ctx.cpltab_id); |
| 715 |
int nAC = (int) LC_GetTableLen(jm, ctx.cpltab_id); |
| 716 |
int i, j; |
| 717 |
|
| 718 |
for(i=0, j=0; i<nAC; i++) |
| 719 |
{ |
| 720 |
while ((j<nJ) && (OBJ_GID(itemsJ[j]->hdr) < theAC[i].gid)) |
| 721 |
j++; |
| 722 |
|
| 723 |
if ((j<nJ) && (OBJ_GID(itemsJ[j]->hdr) == theAC[i].gid)) |
| 724 |
{ |
| 725 |
/* found local object which is AddCpl target */ |
| 726 |
AddCoupling(context, itemsJ[j]->hdr, theAC[i].proc, theAC[i].prio); |
| 727 |
|
| 728 |
# if DebugJoin<=1 |
| 729 |
printf("%4d: Phase3 execute AddCpl(%08x,%d,%d) (from %d).\n", |
| 730 |
context.me(), |
| 731 |
OBJ_GID(itemsJ[j]->hdr), theAC[i].proc, theAC[i].prio, |
| 732 |
LC_MsgGetProc(jm)); |
| 733 |
# endif |
| 734 |
} |
| 735 |
else |
| 736 |
{ |
| 737 |
/* this should never happen. AddCpl send for unknown obj. */ |
| 738 |
assert(0); |
| 739 |
} |
| 740 |
} |
| 741 |
} |
| 742 |
} |
| 743 |
|
| 744 |
|
| 745 |
|
| 746 |
|
| 747 |
/****************************************************************************/ |
| 748 |
/* */ |
| 749 |
/* Function: DDD_JoinEnd */ |
| 750 |
/* */ |
| 751 |
/****************************************************************************/ |
| 752 |
|
| 753 |
/** |
| 754 |
End of join phase. |
| 755 |
This function starts the actual join process. After a call to |
| 756 |
this function (on all processors) all {\bf Join}-commands since |
| 757 |
the last call to \funk{JoinBegin} are executed. This involves |
| 758 |
a set of local communications between the processors. |
| 759 |
*/ |
| 760 |
|
| 761 |
DDD_RET DDD_JoinEnd(DDD::DDDContext& context) |
| 762 |
{ |
| 763 |
auto& ctx = context.joinContext(); |
| 764 |
|
| 765 |
int obsolete, nRecvMsgs1, nRecvMsgs2, nRecvMsgs3; |
| 766 |
JOINMSG1 *sendMsgs1=NULL, *sm1=NULL; |
| 767 |
JOINMSG2 *sendMsgs2=NULL, *sm2=NULL; |
| 768 |
JOINMSG3 *sendMsgs3=NULL, *sm3=NULL; |
| 769 |
LC_MSGHANDLE *recvMsgs1=NULL, *recvMsgs2=NULL, *recvMsgs3=NULL; |
| 770 |
size_t sendMem=0, recvMem=0; |
| 771 |
JIPartner *joinObjs = NULL; |
| 772 |
int nJoinObjs; |
| 773 |
const auto& nCpls = context.couplingContext().nCpls; |
| 774 |
|
| 775 |
|
| 776 |
|
| 777 |
STAT_SET_MODULE(DDD_MODULE_JOIN); |
| 778 |
STAT_ZEROALL; |
| 779 |
|
| 780 |
/* step mode and check whether call to JoinEnd is valid */ |
| 781 |
if (!JoinStepMode(context, JoinMode::JMODE_CMDS)) |
| 782 |
DUNE_THROW(Dune::Exception, "DDD_JoinEnd() aborted"); |
| 783 |
|
| 784 |
|
| 785 |
/* |
| 786 |
PREPARATION PHASE |
| 787 |
*/ |
| 788 |
/* get sorted array of JIJoin-items */ |
| 789 |
std::vector<JIJoin*> arrayJIJoin = JIJoinSet_GetArray(reinterpret_cast<JIJoinSet*>(ctx.setJIJoin)); |
| 790 |
obsolete = JIJoinSet_GetNDiscarded(reinterpret_cast<JIJoinSet*>(ctx.setJIJoin)); |
| 791 |
|
| 792 |
|
| 793 |
/* |
| 794 |
COMMUNICATION PHASE 1 |
| 795 |
all processors, where JoinObj-commands have been issued, |
| 796 |
send information about these commands to the target |
| 797 |
processors together with the GID of the objects on the |
| 798 |
target procs and the local priority. |
| 799 |
*/ |
| 800 |
STAT_RESET; |
| 801 |
/* prepare msgs for JIJoin-items */ |
| 802 |
PreparePhase1Msgs(context, arrayJIJoin, &sendMsgs1, &sendMem); |
| 803 |
/* DisplayMemResources(); */ |
| 804 |
|
| 805 |
/* init communication topology */ |
| 806 |
nRecvMsgs1 = LC_Connect(context, ctx.phase1msg_t); |
| 807 |
STAT_TIMER(T_JOIN_PREP_MSGS); |
| 808 |
|
| 809 |
STAT_RESET; |
| 810 |
/* build phase1 msgs on sender side and start send */ |
| 811 |
PackPhase1Msgs(context, sendMsgs1); |
| 812 |
STAT_TIMER(T_JOIN_PACK_SEND); |
| 813 |
|
| 814 |
|
| 815 |
/* |
| 816 |
now messages are in the net, use spare time |
| 817 |
*/ |
| 818 |
STAT_RESET; |
| 819 |
/* get sorted list of local objects with couplings */ |
| 820 |
std::vector<DDD_HDR> localCplObjs = LocalCoupledObjectsList(context); |
| 821 |
|
| 822 |
if (obsolete>0) |
| 823 |
{ |
| 824 |
if (DDD_GetOption(context, OPT_INFO_JOIN) & JOIN_SHOW_OBSOLETE) |
| 825 |
{ |
| 826 |
int all = JIJoinSet_GetNItems(reinterpret_cast<JIJoinSet*>(ctx.setJIJoin)); |
| 827 |
|
| 828 |
using std::setw; |
| 829 |
Dune::dwarn |
| 830 |
<< "DDD MESG [" << setw(3) << context.me() << "]: " << setw(4) << obsolete |
| 831 |
<< " from " << setw(4) << all << " join-cmds obsolete.\n"; |
| 832 |
} |
| 833 |
} |
| 834 |
STAT_TIMER(T_JOIN); |
| 835 |
|
| 836 |
|
| 837 |
/* |
| 838 |
nothing more to do until incoming messages arrive |
| 839 |
*/ |
| 840 |
|
| 841 |
/* display information about send-messages on lowcomm-level */ |
| 842 |
if (DDD_GetOption(context, OPT_INFO_JOIN) & JOIN_SHOW_MSGSALL) |
| 843 |
{ |
| 844 |
DDD_SyncAll(context); |
| 845 |
if (context.isMaster()) |
| 846 |
Dune::dwarn << "DDD JOIN_SHOW_MSGSALL: Phase1Msg.Send\n"; |
| 847 |
LC_PrintSendMsgs(context); |
| 848 |
} |
| 849 |
|
| 850 |
|
| 851 |
/* wait for communication-completion (send AND receive) */ |
| 852 |
STAT_RESET; |
| 853 |
recvMsgs1 = LC_Communicate(context); |
| 854 |
STAT_TIMER(T_JOIN_WAIT_RECV); |
| 855 |
|
| 856 |
|
| 857 |
/* display information about message buffer sizes */ |
| 858 |
if (DDD_GetOption(context, OPT_INFO_JOIN) & JOIN_SHOW_MEMUSAGE) |
| 859 |
{ |
| 860 |
int k; |
| 861 |
|
| 862 |
/* sum up sizes of receive mesg buffers */ |
| 863 |
for(k=0; k<nRecvMsgs1; k++) |
| 864 |
{ |
| 865 |
recvMem += LC_GetBufferSize(recvMsgs1[k]); |
| 866 |
} |
| 867 |
|
| 868 |
using std::setw; |
| 869 |
Dune::dwarn |
| 870 |
<< "DDD MESG [" << setw(3) << context.me() << "]: SHOW_MEM msgs " |
| 871 |
<< " send=" << setw(10) << sendMem |
| 872 |
<< " recv=" << setw(10) << recvMem |
| 873 |
<< " all=" << setw(10) << (sendMem+recvMem) << "\n"; |
| 874 |
} |
| 875 |
|
| 876 |
/* display information about recv-messages on lowcomm-level */ |
| 877 |
if (DDD_GetOption(context, OPT_INFO_JOIN) & JOIN_SHOW_MSGSALL) |
| 878 |
{ |
| 879 |
DDD_SyncAll(context); |
| 880 |
if (context.isMaster()) |
| 881 |
Dune::dwarn << "DDD JOIN_SHOW_MSGSALL: Phase1Msg.Recv\n"; |
| 882 |
LC_PrintRecvMsgs(context); |
| 883 |
} |
| 884 |
|
| 885 |
/* unpack messages */ |
| 886 |
STAT_RESET; |
| 887 |
UnpackPhase1Msgs(context, recvMsgs1, nRecvMsgs1, localCplObjs.data(), nCpls, |
| 888 |
&joinObjs, &nJoinObjs); |
| 889 |
LC_Cleanup(context); |
| 890 |
STAT_TIMER(T_JOIN_UNPACK); |
| 891 |
|
| 892 |
|
| 893 |
|
| 894 |
|
| 895 |
|
| 896 |
/* |
| 897 |
COMMUNICATION PHASE 2 |
| 898 |
all processors which received notification of JoinObj-commands |
| 899 |
during phase 1 send AddCpl-requests to all copies of DDD objects, |
| 900 |
for which Joins had been issued remotely. |
| 901 |
*/ |
| 902 |
/* get sorted array of JIAddCpl-items */ |
| 903 |
std::vector<JIAddCpl*> arrayJIAddCpl2 = JIAddCplSet_GetArray(reinterpret_cast<JIAddCplSet*>(ctx.setJIAddCpl2)); |
| 904 |
|
| 905 |
STAT_RESET; |
| 906 |
/* prepare msgs for JIAddCpl-items */ |
| 907 |
PreparePhase2Msgs(context, arrayJIAddCpl2, &sendMsgs2, &sendMem); |
| 908 |
/* DisplayMemResources(); */ |
| 909 |
|
| 910 |
/* init communication topology */ |
| 911 |
nRecvMsgs2 = LC_Connect(context, ctx.phase2msg_t); |
| 912 |
STAT_TIMER(T_JOIN_PREP_MSGS); |
| 913 |
|
| 914 |
STAT_RESET; |
| 915 |
/* build phase2 msgs on sender side and start send */ |
| 916 |
PackPhase2Msgs(context, sendMsgs2); |
| 917 |
STAT_TIMER(T_JOIN_PACK_SEND); |
| 918 |
|
| 919 |
/* |
| 920 |
now messages are in the net, use spare time |
| 921 |
*/ |
| 922 |
|
| 923 |
/* reorder Join-commands according to new_gid */ |
| 924 |
/* this ordering is needed in UnpackPhase3 */ |
| 925 |
if (arrayJIJoin.size() > 1) |
| 926 |
{ |
| 927 |
std::sort( |
| 928 |
arrayJIJoin.begin(), arrayJIJoin.end(), |
| 929 |
[](const JIJoin* a, const JIJoin* b) { |
| 930 |
return a->new_gid < b->new_gid; |
| 931 |
}); |
| 932 |
} |
| 933 |
|
| 934 |
|
| 935 |
/* |
| 936 |
nothing more to do until incoming messages arrive |
| 937 |
*/ |
| 938 |
|
| 939 |
/* display information about send-messages on lowcomm-level */ |
| 940 |
if (DDD_GetOption(context, OPT_INFO_JOIN) & JOIN_SHOW_MSGSALL) |
| 941 |
{ |
| 942 |
DDD_SyncAll(context); |
| 943 |
if (context.isMaster()) |
| 944 |
Dune::dwarn <<"DDD JOIN_SHOW_MSGSALL: Phase2Msg.Send\n"; |
| 945 |
LC_PrintSendMsgs(context); |
| 946 |
} |
| 947 |
|
| 948 |
|
| 949 |
/* wait for communication-completion (send AND receive) */ |
| 950 |
STAT_RESET; |
| 951 |
recvMsgs2 = LC_Communicate(context); |
| 952 |
STAT_TIMER(T_JOIN_WAIT_RECV); |
| 953 |
|
| 954 |
|
| 955 |
/* display information about message buffer sizes */ |
| 956 |
if (DDD_GetOption(context, OPT_INFO_JOIN) & JOIN_SHOW_MEMUSAGE) |
| 957 |
{ |
| 958 |
int k; |
| 959 |
|
| 960 |
/* sum up sizes of receive mesg buffers */ |
| 961 |
for(k=0; k<nRecvMsgs2; k++) |
| 962 |
{ |
| 963 |
recvMem += LC_GetBufferSize(recvMsgs2[k]); |
| 964 |
} |
| 965 |
|
| 966 |
using std::setw; |
| 967 |
Dune::dwarn |
| 968 |
<< "DDD MESG [" << setw(3) << context.me() << "]: SHOW_MEM msgs " |
| 969 |
<< " send=" << setw(10) << sendMem |
| 970 |
<< " recv=" << setw(10) << recvMem |
| 971 |
<< " all=" << setw(10) << (sendMem+recvMem) << "\n"; |
| 972 |
} |
| 973 |
|
| 974 |
/* display information about recv-messages on lowcomm-level */ |
| 975 |
if (DDD_GetOption(context, OPT_INFO_JOIN) & JOIN_SHOW_MSGSALL) |
| 976 |
{ |
| 977 |
DDD_SyncAll(context); |
| 978 |
if (context.isMaster()) |
| 979 |
Dune::dwarn << "DDD JOIN_SHOW_MSGSALL: Phase2Msg.Recv\n"; |
| 980 |
LC_PrintRecvMsgs(context); |
| 981 |
} |
| 982 |
|
| 983 |
/* unpack messages */ |
| 984 |
STAT_RESET; |
| 985 |
UnpackPhase2Msgs(context, recvMsgs2, nRecvMsgs2, joinObjs, nJoinObjs, |
| 986 |
localCplObjs.data(), nCpls); |
| 987 |
|
| 988 |
LC_Cleanup(context); |
| 989 |
STAT_TIMER(T_JOIN_UNPACK); |
| 990 |
|
| 991 |
for(; sendMsgs2!=NULL; sendMsgs2=sm2) |
| 992 |
{ |
| 993 |
sm2 = sendMsgs2->next; |
| 994 |
delete sendMsgs2; |
| 995 |
} |
| 996 |
|
| 997 |
|
| 998 |
|
| 999 |
|
| 1000 |
|
| 1001 |
|
| 1002 |
/* |
| 1003 |
COMMUNICATION PHASE 3 |
| 1004 |
all processors which received notification of JoinObj-commands |
| 1005 |
during phase 1 send AddCpl-requests to the procs where the |
| 1006 |
JoinObj-commands have been issued. One AddCpl-request is sent |
| 1007 |
for each cpl in the local objects coupling list. One AddCpl-request |
| 1008 |
is sent for each AddCpl-request received during phase 2. |
| 1009 |
(i.e., two kinds of AddCpl-requests are send to the processors |
| 1010 |
on which the JoinObj-commands have been issued. |
| 1011 |
*/ |
| 1012 |
/* get sorted array of JIAddCpl-items */ |
| 1013 |
std::vector<JIAddCpl*> arrayJIAddCpl3 = JIAddCplSet_GetArray(reinterpret_cast<JIAddCplSet*>(ctx.setJIAddCpl3)); |
| 1014 |
|
| 1015 |
STAT_RESET; |
| 1016 |
/* prepare msgs for JIAddCpl-items */ |
| 1017 |
PreparePhase3Msgs(context, arrayJIAddCpl3, &sendMsgs3, &sendMem); |
| 1018 |
/* DisplayMemResources(); */ |
| 1019 |
|
| 1020 |
/* init communication topology */ |
| 1021 |
nRecvMsgs3 = LC_Connect(context, ctx.phase3msg_t); |
| 1022 |
STAT_TIMER(T_JOIN_PREP_MSGS); |
| 1023 |
|
| 1024 |
STAT_RESET; |
| 1025 |
/* build phase3 msgs on sender side and start send */ |
| 1026 |
PackPhase3Msgs(context, sendMsgs3); |
| 1027 |
STAT_TIMER(T_JOIN_PACK_SEND); |
| 1028 |
|
| 1029 |
/* |
| 1030 |
now messages are in the net, use spare time |
| 1031 |
*/ |
| 1032 |
/* ... */ |
| 1033 |
|
| 1034 |
/* |
| 1035 |
nothing more to do until incoming messages arrive |
| 1036 |
*/ |
| 1037 |
|
| 1038 |
/* display information about send-messages on lowcomm-level */ |
| 1039 |
if (DDD_GetOption(context, OPT_INFO_JOIN) & JOIN_SHOW_MSGSALL) |
| 1040 |
{ |
| 1041 |
DDD_SyncAll(context); |
| 1042 |
if (context.isMaster()) |
| 1043 |
Dune::dwarn << "DDD JOIN_SHOW_MSGSALL: Phase3Msg.Send\n"; |
| 1044 |
LC_PrintSendMsgs(context); |
| 1045 |
} |
| 1046 |
|
| 1047 |
|
| 1048 |
/* wait for communication-completion (send AND receive) */ |
| 1049 |
STAT_RESET; |
| 1050 |
recvMsgs3 = LC_Communicate(context); |
| 1051 |
STAT_TIMER(T_JOIN_WAIT_RECV); |
| 1052 |
|
| 1053 |
|
| 1054 |
/* display information about message buffer sizes */ |
| 1055 |
if (DDD_GetOption(context, OPT_INFO_JOIN) & JOIN_SHOW_MEMUSAGE) |
| 1056 |
{ |
| 1057 |
int k; |
| 1058 |
|
| 1059 |
/* sum up sizes of receive mesg buffers */ |
| 1060 |
for(k=0; k<nRecvMsgs3; k++) |
| 1061 |
{ |
| 1062 |
recvMem += LC_GetBufferSize(recvMsgs3[k]); |
| 1063 |
} |
| 1064 |
|
| 1065 |
using std::setw; |
| 1066 |
Dune::dwarn |
| 1067 |
<< "DDD MESG [" << setw(3) << context.me() << "]: SHOW_MEM msgs " |
| 1068 |
<< " send=" << setw(10) << sendMem |
| 1069 |
<< " recv=" << setw(10) << recvMem |
| 1070 |
<< " all=" << setw(10) << (sendMem+recvMem) << "\n"; |
| 1071 |
} |
| 1072 |
|
| 1073 |
/* display information about recv-messages on lowcomm-level */ |
| 1074 |
if (DDD_GetOption(context, OPT_INFO_JOIN) & JOIN_SHOW_MSGSALL) |
| 1075 |
{ |
| 1076 |
DDD_SyncAll(context); |
| 1077 |
if (context.isMaster()) |
| 1078 |
Dune::dwarn << "DDD JOIN_SHOW_MSGSALL: Phase3Msg.Recv\n"; |
| 1079 |
LC_PrintRecvMsgs(context); |
| 1080 |
} |
| 1081 |
|
| 1082 |
/* unpack messages */ |
| 1083 |
STAT_RESET; |
| 1084 |
UnpackPhase3Msgs(context, recvMsgs3, nRecvMsgs3, arrayJIJoin); |
| 1085 |
LC_Cleanup(context); |
| 1086 |
STAT_TIMER(T_JOIN_UNPACK); |
| 1087 |
|
| 1088 |
for(; sendMsgs3!=NULL; sendMsgs3=sm3) |
| 1089 |
{ |
| 1090 |
sm3 = sendMsgs3->next; |
| 1091 |
delete sendMsgs3; |
| 1092 |
} |
| 1093 |
|
| 1094 |
|
| 1095 |
|
| 1096 |
|
| 1097 |
|
| 1098 |
|
| 1099 |
/* |
| 1100 |
free temporary storage |
| 1101 |
*/ |
| 1102 |
JIJoinSet_Reset(reinterpret_cast<JIJoinSet*>(ctx.setJIJoin)); |
| 1103 |
|
| 1104 |
JIAddCplSet_Reset(reinterpret_cast<JIAddCplSet*>(ctx.setJIAddCpl2)); |
| 1105 |
|
| 1106 |
JIAddCplSet_Reset(reinterpret_cast<JIAddCplSet*>(ctx.setJIAddCpl3)); |
| 1107 |
|
| 1108 |
if (joinObjs!=NULL) |
| 1109 |
delete[] joinObjs; |
| 1110 |
|
| 1111 |
for(; sendMsgs1!=NULL; sendMsgs1=sm1) |
| 1112 |
{ |
| 1113 |
sm1 = sendMsgs1->next; |
| 1114 |
delete sendMsgs1; |
| 1115 |
} |
| 1116 |
|
| 1117 |
|
| 1118 |
|
| 1119 |
# if DebugJoin<=4 |
| 1120 |
Dune::dverb << "JoinEnd, before IFAllFromScratch().\n"; |
| 1121 |
# endif |
| 1122 |
|
| 1123 |
/* re-create all interfaces and step JMODE */ |
| 1124 |
STAT_RESET; |
| 1125 |
IFAllFromScratch(context); |
| 1126 |
STAT_TIMER(T_JOIN_BUILD_IF); |
| 1127 |
|
| 1128 |
|
| 1129 |
JoinStepMode(context, JoinMode::JMODE_BUSY); |
| 1130 |
|
| 1131 |
return(DDD_RET_OK); |
| 1132 |
} |
| 1133 |
|
| 1134 |
|
| 1135 |
|
| 1136 |
|
| 1137 |
|
| 1138 |
/****************************************************************************/ |
| 1139 |
/* */ |
| 1140 |
/* Function: DDD_JoinObj */ |
| 1141 |
/* */ |
| 1142 |
/****************************************************************************/ |
| 1143 |
|
| 1144 |
/** |
| 1145 |
Join local object with a distributed object. |
| 1146 |
|
| 1147 |
\todoTBC |
| 1148 |
|
| 1149 |
@param hdr DDD local object which should be joined. |
| 1150 |
*/ |
| 1151 |
|
| 1152 |
|
| 1153 |
|
| 1154 |
void DDD_JoinObj(DDD::DDDContext& context, DDD_HDR hdr, DDD_PROC dest, DDD_GID new_gid) |
| 1155 |
{ |
| 1156 |
auto& ctx = context.joinContext(); |
| 1157 |
const auto procs = context.procs(); |
| 1158 |
|
| 1159 |
if (!ddd_JoinActive(context)) |
| 1160 |
DUNE_THROW(Dune::Exception, "Missing DDD_JoinBegin()"); |
| 1161 |
|
| 1162 |
if (dest>=procs) |
| 1163 |
DUNE_THROW(Dune::Exception, |
| 1164 |
"cannot join " << OBJ_GID(hdr) << " with " << new_gid |
| 1165 |
<< " on processor " << dest << " (procs=" << procs << ")"); |
| 1166 |
|
| 1167 |
if (dest==context.me()) |
| 1168 |
DUNE_THROW(Dune::Exception, |
| 1169 |
"cannot join " << OBJ_GID(hdr) << " with myself"); |
| 1170 |
|
| 1171 |
if (ObjHasCpl(context, hdr)) |
| 1172 |
DUNE_THROW(Dune::Exception, |
| 1173 |
"cannot join " << OBJ_GID(hdr) |
| 1174 |
<< ", object already distributed"); |
| 1175 |
|
| 1176 |
|
| 1177 |
|
| 1178 |
JIJoin* ji = JIJoinSet_NewItem(reinterpret_cast<JIJoinSet*>(ctx.setJIJoin)); |
| 1179 |
ji->hdr = hdr; |
| 1180 |
ji->dest = dest; |
| 1181 |
ji->new_gid = new_gid; |
| 1182 |
|
| 1183 |
if (! JIJoinSet_ItemOK(reinterpret_cast<JIJoinSet*>(ctx.setJIJoin))) |
| 1184 |
return; |
| 1185 |
|
| 1186 |
# if DebugJoin<=2 |
| 1187 |
Dune:dvverb << "DDD_JoinObj " << OBJ_GID(hdr) |
| 1188 |
<< ", dest=" << dest << ", new_gid=" << new_gid << "\n"; |
| 1189 |
# endif |
| 1190 |
} |
| 1191 |
|
| 1192 |
|
| 1193 |
|
| 1194 |
|
| 1195 |
/****************************************************************************/ |
| 1196 |
/* */ |
| 1197 |
/* Function: DDD_JoinBegin */ |
| 1198 |
/* */ |
| 1199 |
/****************************************************************************/ |
| 1200 |
|
| 1201 |
/** |
| 1202 |
Starts join phase. |
| 1203 |
A call to this function establishes a global join operation. |
| 1204 |
It must be issued on all processors. After this call an arbitrary |
| 1205 |
series of {\bf Join}-commands may be issued. The global transfer operation |
| 1206 |
is carried out via a \funk{JoinEnd} call on each processor. |
| 1207 |
*/ |
| 1208 |
|
| 1209 |
void DDD_JoinBegin(DDD::DDDContext& context) |
| 1210 |
{ |
| 1211 |
/* step mode and check whether call to JoinBegin is valid */ |
| 1212 |
if (!JoinStepMode(context, JoinMode::JMODE_IDLE)) |
| 1213 |
DUNE_THROW(Dune::Exception, "DDD_JoinBegin() aborted"); |
| 1214 |
} |
| 1215 |
|
| 1216 |
|
| 1217 |
|
| 1218 |
END_UGDIM_NAMESPACE |