Debian Codegraph @

dune/uggrid/parallel/ddd/xfer/cplmsg.cc

raw
1 // SPDX-FileCopyrightText: Copyright © DUNE Project contributors, see file LICENSE.md in module root
2 // SPDX-License-Identifier: LGPL-2.1-or-later
3 // -*- tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 2 -*-
4 // vi: set et ts=4 sw=2 sts=2:
5 /****************************************************************************/
6 /* */
7 /* File: cplmsg.c */
8 /* */
9 /* Purpose: ddd object transfer: */
10 /* last messages in order to create coupling consistency. */
11 /* */
12 /* Author: Klaus Birken */
13 /* Rechenzentrum Uni Stuttgart */
14 /* Universitaet Stuttgart */
15 /* Allmandring 30 */
16 /* 70550 Stuttgart */
17 /* internet: [email protected] */
18 /* */
19 /* History: 960712 kb created */
20 /* 960718 kb introduced lowcomm-layer (sets of messages) */
21 /* */
22 /* Remarks: */
23 /* */
24 /****************************************************************************/
25
26 /****************************************************************************/
27 /* */
28 /* include files */
29 /* system include files */
30 /* application include files */
31 /* */
32 /****************************************************************************/
33
34 /* standard C library */
35 #include <config.h>
36 #include <cstdlib>
37 #include <cstdio>
38
39 #include <forward_list>
40 #include <iomanip>
41 #include <iostream>
42 #include <sstream>
43
44 #include <dune/common/stdstreams.hh>
45
46 #include <dune/uggrid/parallel/ddd/dddcontext.hh>
47
48 #include <dune/uggrid/parallel/ddd/dddi.h>
49 #include "xfer.h"
50
51
52 USING_UG_NAMESPACE
53 using namespace PPIF;
54
55 START_UGDIM_NAMESPACE
56
57 /****************************************************************************/
58 /* */
59 /* data structures */
60 /* */
61 /****************************************************************************/
62
63 /* CPLMSG: complete description of message on sender side */
64
65 struct CPLMSG
66 {
67 DDD_PROC proc;
68
69 XIDelCpl **xferDelCpl = nullptr;
70 int nDelCpl = 0;
71
72 XIModCpl **xferModCpl = nullptr;
73 int nModCpl = 0;
74
75 XIAddCpl **xferAddCpl = nullptr;
76 int nAddCpl = 0;
77
78
79 /* lowcomm message handle */
80 LC_MSGHANDLE msg_h;
81
82 CPLMSG(DDD_PROC dest)
83 : proc(dest)
84 {}
85 };
86
87 using CplmsgList = std::forward_list<CPLMSG>;
88
89 /****************************************************************************/
90 /* */
91 /* routines */
92 /* */
93 /****************************************************************************/
94
95
96 void CplMsgInit(DDD::DDDContext& context)
97 {
98 auto& ctx = context.cplmsgContext();
99
100 ctx.cplmsg_t = LC_NewMsgType(context, "CplMsg");
101 ctx.delcpl_id = LC_NewMsgTable("DelCpl", ctx.cplmsg_t, sizeof(TEDelCpl));
102 ctx.modcpl_id = LC_NewMsgTable("ModCpl", ctx.cplmsg_t, sizeof(TEModCpl));
103 ctx.addcpl_id = LC_NewMsgTable("AddCpl", ctx.cplmsg_t, sizeof(TEAddCpl));
104 }
105
106
107 void CplMsgExit(DDD::DDDContext&)
108 {}
109
110
111 /****************************************************************************/
112
113 static
114 std::pair<int, CplmsgList>
115 PrepareCplMsgs (
116 DDD::DDDContext& context,
117 XIDelCpl **itemsDC, int nDC,
118 XIModCpl **itemsMC, int nMC,
119 XIAddCpl **itemsAC, int nAC)
120 {
121 auto& ctx = context.cplmsgContext();
122 const auto procs = context.procs();
123
124 # if DebugCplMsg<=3
125 Dune::dverb << "PrepareCplMsgs, nXIDelCpl=" << nDC << " nXIModCpl=" << nMC
126 << " nXIAddCpl=" << nAC << "\n";
127 # endif
128
129 CplmsgList msgs;
130 int nMsgs = 0;
131
132 auto msgForProc = [&](DDD_PROC dest) -> CPLMSG& {
133 if (msgs.empty() or msgs.front().proc != dest) {
134 msgs.emplace_front(dest);
135 ++nMsgs;
136 }
137 return msgs.front();
138 };
139
140 /*
141 run through all tables simultaneously,
142 each time a new proc-nr is encountered in one of these
143 tables, create a new CPLMSG item.
144
145 (the lists have been sorted according to proc-nr previously.)
146 */
147
148 int iDC=0, iMC=0, iAC=0;
149 while (iDC<nDC || iMC<nMC || iAC<nAC)
150 {
151 DDD_PROC pDC = (iDC<nDC) ? itemsDC[iDC]->to : procs;
152 DDD_PROC pMC = (iMC<nMC) ? itemsMC[iMC]->to : procs;
153 DDD_PROC pAC = (iAC<nAC) ? itemsAC[iAC]->to : procs;
154
155 /* check DelCpl-items */
156 if (pDC<=pMC && pDC<=pAC && pDC<procs)
157 {
158 int i;
159
160 CPLMSG& xm = msgForProc(pDC);
161 xm.xferDelCpl = itemsDC+iDC;
162 for(i=iDC; i<nDC && itemsDC[i]->to==pDC; i++)
163 ;
164
165 xm.nDelCpl = i-iDC;
166 iDC = i;
167 }
168
169 /* check ModCpl-items */
170 if (pMC<=pDC && pMC<=pAC && pMC<procs)
171 {
172 int i;
173
174 CPLMSG& xm = msgForProc(pMC);
175 xm.xferModCpl = itemsMC+iMC;
176 for(i=iMC; i<nMC && itemsMC[i]->to==pMC; i++)
177 ;
178
179 xm.nModCpl = i-iMC;
180 iMC = i;
181 }
182
183 /* check AddCpl-items */
184 if (pAC<=pDC && pAC<=pMC && pAC<procs)
185 {
186 int i;
187
188 CPLMSG& xm = msgForProc(pAC);
189 xm.xferAddCpl = itemsAC+iAC;
190 for(i=iAC; i<nAC && itemsAC[i]->to==pAC; i++)
191 ;
192
193 xm.nAddCpl = i-iAC;
194 iAC = i;
195 }
196
197 if (pDC==procs) iDC = nDC;
198 if (pMC==procs) iMC = nMC;
199 if (pAC==procs) iAC = nAC;
200 }
201
202
203 /* initiate send messages */
204 for(auto& xm : msgs)
205 {
206 /* create new send message */
207 xm.msg_h = LC_NewSendMsg(context, ctx.cplmsg_t, xm.proc);
208
209 /* init tables inside message */
210 LC_SetTableSize(xm.msg_h, ctx.delcpl_id, xm.nDelCpl);
211 LC_SetTableSize(xm.msg_h, ctx.modcpl_id, xm.nModCpl);
212 LC_SetTableSize(xm.msg_h, ctx.addcpl_id, xm.nAddCpl);
213
214 /* prepare message for sending away */
215 LC_MsgPrepareSend(context, xm.msg_h);
216 }
217
218 return {nMsgs, std::move(msgs)};
219 }
220
221
222 static void CplMsgSend(DDD::DDDContext& context, const CplmsgList& msgs)
223 {
224 auto& ctx = context.cplmsgContext();
225
226 for(const auto& msg : msgs)
227 {
228 TEDelCpl *arrayDC = (TEDelCpl *)LC_GetPtr(msg.msg_h, ctx.delcpl_id);
229 TEModCpl *arrayMC = (TEModCpl *)LC_GetPtr(msg.msg_h, ctx.modcpl_id);
230 TEAddCpl *arrayAC = (TEAddCpl *)LC_GetPtr(msg.msg_h, ctx.addcpl_id);
231
232 /* copy data into message */
233 for(int i=0; i < msg.nDelCpl; i++)
234 {
235 arrayDC[i] = msg.xferDelCpl[i]->te;
236 }
237 for(int i=0; i < msg.nModCpl; i++)
238 {
239 arrayMC[i] = msg.xferModCpl[i]->te;
240 }
241 for(int i=0; i < msg.nAddCpl; i++)
242 {
243 arrayAC[i] = msg.xferAddCpl[i]->te;
244 }
245
246 /* schedule message for send */
247 LC_MsgSend(context, msg.msg_h);
248 }
249 }
250
251
252
253 /****************************************************************************/
254
255
256 static void CplMsgUnpackSingle (DDD::DDDContext& context, LC_MSGHANDLE xm,
257 DDD_HDR *localCplObjs, int nLCO)
258 {
259 auto& ctx = context.cplmsgContext();
260
261 TEDelCpl *theDelCpl;
262 TEModCpl *theModCpl;
263 TEAddCpl *theAddCpl;
264 int i, j, nDelCpl, nModCpl, nAddCpl;
265 DDD_PROC proc = LC_MsgGetProc(xm);
266
267 /* get number and address of del-items */
268 nDelCpl = (int) LC_GetTableLen(xm, ctx.delcpl_id);
269 nModCpl = (int) LC_GetTableLen(xm, ctx.modcpl_id);
270 nAddCpl = (int) LC_GetTableLen(xm, ctx.addcpl_id);
271 theDelCpl = (TEDelCpl *) LC_GetPtr(xm, ctx.delcpl_id);
272 theModCpl = (TEModCpl *) LC_GetPtr(xm, ctx.modcpl_id);
273 theAddCpl = (TEAddCpl *) LC_GetPtr(xm, ctx.addcpl_id);
274
275
276 /* modify couplings according to mod-list */
277 for(i=0, j=0; i<nModCpl; i++)
278 {
279 while ((j<nLCO) && (OBJ_GID(localCplObjs[j]) < theModCpl[i].gid))
280 j++;
281
282 if ((j<nLCO) && (OBJ_GID(localCplObjs[j])==theModCpl[i].gid))
283 {
284 ModCoupling(context, localCplObjs[j], proc, theModCpl[i].prio);
285 }
286 }
287
288
289 /* delete couplings according to del-list */
290 for(i=0, j=0; i<nDelCpl; i++)
291 {
292 while ((j<nLCO) && (OBJ_GID(localCplObjs[j]) < theDelCpl[i].gid))
293 j++;
294
295 if ((j<nLCO) && (OBJ_GID(localCplObjs[j])==theDelCpl[i].gid))
296 {
297 DelCoupling(context, localCplObjs[j], proc);
298 }
299 }
300
301
302 /* add couplings according to add-list */
303 for(i=0, j=0; i<nAddCpl; i++)
304 {
305 while ((j<nLCO) && (OBJ_GID(localCplObjs[j]) < theAddCpl[i].gid))
306 j++;
307
308 if ((j<nLCO) && (OBJ_GID(localCplObjs[j])==theAddCpl[i].gid))
309 {
310 AddCoupling(context, localCplObjs[j], theAddCpl[i].proc, theAddCpl[i].prio);
311 }
312 }
313 }
314
315
316 /****************************************************************************/
317
318
319 static void CplMsgDisplay (DDD::DDDContext& context, const char *comment, LC_MSGHANDLE xm)
320 {
321 using std::setw;
322
323 std::ostream& out = std::cout;
324 auto& ctx = context.cplmsgContext();
325 TEDelCpl *theDelCpl;
326 TEModCpl *theModCpl;
327 TEAddCpl *theAddCpl;
328 int i, proc = LC_MsgGetProc(xm);
329 int lenDelCpl = (int) LC_GetTableLen(xm, ctx.delcpl_id);
330 int lenModCpl = (int) LC_GetTableLen(xm, ctx.modcpl_id);
331 int lenAddCpl = (int) LC_GetTableLen(xm, ctx.addcpl_id);
332
333 std::ostringstream prefixStream;
334 prefixStream
335 << " " << setw(3) << context.me() << "-" << comment << "-" << setw(3) << proc << " ";
336 const std::string& prefix = prefixStream.str();
337
338 /* get table addresses inside message */
339 theDelCpl = (TEDelCpl *) LC_GetPtr(xm, ctx.delcpl_id);
340 theModCpl = (TEModCpl *) LC_GetPtr(xm, ctx.modcpl_id);
341 theAddCpl = (TEAddCpl *) LC_GetPtr(xm, ctx.addcpl_id);
342
343
344 out << prefix << " 04 DelCpl.size=" << setw(5) << lenDelCpl << "\n";
345 out << prefix << " 05 ModCpl.size=" << setw(5) << lenModCpl << "\n";
346 out << prefix << " 06 AddCpl.size=" << setw(5) << lenAddCpl << "\n";
347
348 for(i=0; i<lenDelCpl; i++)
349 out << prefix << " 14 delcpl " << setw(4) << i << " - "
350 << theDelCpl[i].gid << "\n";
351
352 for(i=0; i<lenModCpl; i++)
353 out << prefix << " 15 modcpl " << setw(4) << i << " - "
354 << theModCpl[i].gid << " " << setw(3) << theModCpl[i].prio << "\n";
355
356 for(i=0; i<lenAddCpl; i++)
357 out << prefix << " 16 addcpl " << setw(4) << i << " - "
358 << theAddCpl[i].gid << " " << setw(4) << theAddCpl[i].proc
359 << " " << setw(3) << theAddCpl[i].prio << "\n";
360 }
361
362
363 /****************************************************************************/
364
365
366 /*
367 ...
368
369 localCplObjs is a sorted list of all objects with coupling lists.
370 */
371
372 void CommunicateCplMsgs (
373 DDD::DDDContext& context,
374 XIDelCpl **itemsDC, int nDC,
375 XIModCpl **itemsMC, int nMC,
376 XIAddCpl **itemsAC, int nAC,
377 DDD_HDR *localCplObjs, int nLCO)
378 {
379 auto& ctx = context.cplmsgContext();
380
381 /* accumulate messages (one for each partner) */
382 CplmsgList sendMsgs;
383 int nSendMsgs;
384 std::tie(nSendMsgs, sendMsgs) = PrepareCplMsgs(context,
385 itemsDC, nDC,
386 itemsMC, nMC,
387 itemsAC, nAC);
388
389 /* init communication topology */
390 int nRecvMsgs = LC_Connect(context, ctx.cplmsg_t);
391
392 /* build and send messages */
393 CplMsgSend(context, sendMsgs);
394
395
396 #if DebugCplMsg>2
397 if (DDD_GetOption(context, OPT_DEBUG_XFERMESGS)==OPT_ON)
398 #endif
399 {
400 for (const auto& msg : sendMsgs)
401 CplMsgDisplay(context, "CS", msg.msg_h);
402 }
403
404
405 /* display information about send-messages on lowcomm-level */
406 if (DDD_GetOption(context, OPT_INFO_XFER) & XFER_SHOW_MSGSALL)
407 {
408 DDD_SyncAll(context);
409 if (context.isMaster())
410 Dune::dwarn << "DDD XFER_SHOW_MSGSALL: CplMsg.Send\n";
411 LC_PrintSendMsgs(context);
412 }
413
414
415 /* communicate set of messages (send AND receive) */
416 LC_MSGHANDLE* recvMsgs = LC_Communicate(context);
417
418
419 /* display information about recv-messages on lowcomm-level */
420 if (DDD_GetOption(context, OPT_INFO_XFER) & XFER_SHOW_MSGSALL)
421 {
422 DDD_SyncAll(context);
423 if (context.isMaster())
424 Dune::dwarn << "DDD XFER_SHOW_MSGSALL: CplMsg.Recv\n";
425 LC_PrintRecvMsgs(context);
426 }
427
428
429 for(int i=0; i<nRecvMsgs; i++)
430 {
431 CplMsgUnpackSingle(context, recvMsgs[i], localCplObjs, nLCO);
432
433 /*
434 if (DDD_GetOption(context, OPT_DEBUG_XFERMESGS)==OPT_ON)
435 CplMsgDisplay(context, "CR", recvMsgs[i]);
436 */
437 }
438
439
440 /* cleanup low-comm layer */
441 LC_Cleanup(context);
442 }
443
444
445 /****************************************************************************/
446
447 END_UGDIM_NAMESPACE