Debian Codegraph @

dune/uggrid/parallel/ddd/basic/lowcomm.cc

raw
1 // SPDX-FileCopyrightText: Copyright © DUNE Project contributors, see file LICENSE.md in module root
2 // SPDX-License-Identifier: LGPL-2.1-or-later
3 // -*- tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 2 -*-
4 // vi: set et ts=4 sw=2 sts=2:
5 /****************************************************************************/
6 /* */
7 /* File: lowcomm.c */
8 /* */
9 /* Purpose: lowlevel communication layer */
10 /* */
11 /* Author: Klaus Birken */
12 /* Institut fuer Computeranwendungen III */
13 /* Universitaet Stuttgart */
14 /* Pfaffenwaldring 27 */
15 /* 70569 Stuttgart */
16 /* internet: [email protected] */
17 /* */
18 /* History: 960715 kb begin */
19 /* 971007 kb reworked */
20 /* */
21 /* Remarks: */
22 /* This module provides two basic abstractions: */
23 /* - sending of messages without explicit receive calls */
24 /* - message types consisting of a set of components, where */
25 /* components are tables (with entries of equal sizes) and */
26 /* raw data chunks. */
27 /* */
28 /* The LowComm subsystem uses the Notify-subsystem in order to */
29 /* tell receiving processors that corresponding send-calls had */
30 /* been issued. */
31 /* */
32 /* The structure of each message is: */
33 /* */
34 /* description | type */
35 /* -------------------------------------------+--------- */
36 /* magic number | ULONG */
37 /* #components | ULONG */
38 /* offset component1 (from beginning of Msg) | ULONG */
39 /* length component1 (in bytes) | ULONG */
40 /* nItems component1 | ULONG */
41 /* ... | ... */
42 /* offset componentN | ULONG */
43 /* length componentN | ULONG */
44 /* nItems componentN | ULONG */
45 /* component1 */
46 /* ... */
47 /* componentN */
48 /* */
49 /* The LowComm subsystem is able to handle low-memory situations,*/
50 /* where the available memory is not enough for all send- and */
51 /* receive-buffers. See LC_MsgAlloc for details. */
52 /* */
53 /****************************************************************************/
54
55 /****************************************************************************/
56 /* */
57 /* include files */
58 /* system include files */
59 /* application include files */
60 /* */
61 /****************************************************************************/
62
63 /* standard C library */
64 #include <config.h>
65 #include <array>
66 #include <cstdlib>
67 #include <cstdio>
68 #include <cstring>
69 #include <cassert>
70
71 #include <iomanip>
72 #include <iostream>
73 #include <new>
74
75 #include <dune/common/exceptions.hh>
76 #include <dune/common/stdstreams.hh>
77
78 #include <dune/uggrid/parallel/ddd/dddi.h>
79 #include "lowcomm.h"
80 #include "notify.h"
81
82 #include <dune/uggrid/parallel/ddd/dddcontext.hh>
83
84 USING_UG_NAMESPACES
85
86 /* PPIF namespace: */
87 using namespace PPIF;
88
89 #define DebugLowComm 10 /* 0 is all, 10 is off */
90
91 namespace DDD {
92 namespace Basic {
93
94 /****************************************************************************/
95 /* */
96 /* defines in the following order */
97 /* */
98 /* compile time constants defining static data size (i.e. arrays) */
99 /* other constants */
100 /* macros */
101 /* */
102 /****************************************************************************/
103
104
105 /* maximum number of components in a message */
106 #define MAX_COMPONENTS 8
107
108
109 /* dummy magic number for messages */
110 #define MAGIC_DUMMY 0x1234
111
112
113 /* number of entries per chunk in message header */
114 #define HDR_ENTRIES_PER_CHUNK 3
115
116 enum CompType {
117 CT_NONE,
118 CT_TABLE,
119 CT_CHUNK
120 };
121
122
123 /****************************************************************************/
124 /* */
125 /* data structures */
126 /* */
127 /****************************************************************************/
128
129 struct COMP_DESC
130 {
131 const char *name; /* textual description of component */
132 int type; /* type of this component */
133 size_t entry_size; /* size per entry (for tables) */
134 };
135
136
137 struct MSG_TYPE
138 {
139 const char *name; /* textual description of msgtype */
140 int nComps; /* number of components */
141 std::array<COMP_DESC, MAX_COMPONENTS> comp; /* component array */
142
143 MSG_TYPE *next; /* linked list of all message types */
144 };
145
146
147 struct CHUNK_DESC
148 {
149 size_t size; /* size of chunk (in bytes) */
150 ULONG entries; /* number of valid entries (for tables) */
151 ULONG offset; /* offset of chunk in MSG */
152 };
153
154
155 enum MsgState { MSTATE_NEW, MSTATE_FREEZED, MSTATE_ALLOCATED, MSTATE_COMM, MSTATE_READY };
156
157 struct MSG_DESC
158 {
159 int msgState; /* message state of this message (one of MsgState) */
160 MSG_TYPE *msgType; /* message type of this message */
161
162 ULONG magic; /* magic number */
163 CHUNK_DESC *chunks; /* array of chunks */
164
165
166 size_t bufferSize; /* size of message buffer (in bytes) */
167 char *buffer; /* address of message buffer */
168
169
170 MSG_DESC *next; /* linked list inside Send/Recv-queue */
171 DDD_PROC proc;
172 msgid msgId;
173
174 };
175
176
177 struct TABLE_DESC
178 {
179 size_t entry_size; /* size of one table entry */
180 int nMax; /* number of entries in table */
181 int nValid; /* number of valid entries */
182 };
183
184 } /* namespace Basic */
185 } /* namespace DDD */
186
187 /****************************************************************************/
188 /* */
189 /* routines */
190 /* */
191 /****************************************************************************/
192
193 namespace DDD {
194
195 using namespace DDD::Basic;
196
197 /**
198 Initiates LowComm subsystem.
199 This function has to be called exactly once in order
200 to initialize the LowComm subsystem. After a call to
201 this function, the functionality of the LowComm can
202 be used.
203
204 @param aAllocFunc memory allocation function used as the default
205 @param aFreeFunc memory free function used as the default
206 */
207
208 void LC_Init(DDD::DDDContext& context, AllocFunc aAllocFunc, FreeFunc aFreeFunc)
209 {
210 auto& lcContext = context.lowCommContext();
211 lcContext.DefaultAlloc = aAllocFunc;
212 lcContext.DefaultFree = aFreeFunc;
213 LC_SetMemMgrDefault(context);
214 }
215
216
217
218 /**
219 Exits LowComm subsystem.
220 This function frees memory allocated by the LowComm subsystem
221 and shuts down its communication structures.
222 */
223
224 void LC_Exit(DDD::DDDContext& context)
225 {
226 auto& lcContext = context.lowCommContext();
227
228 {
229 auto md = lcContext.FreeMsgDescs;
230 while (md != nullptr) {
231 const auto next = md->next;
232 delete md;
233 md = next;
234 }
235 lcContext.FreeMsgDescs = nullptr;
236 }
237
238 {
239 auto mt = lcContext.MsgTypes;
240 while (mt != nullptr) {
241 const auto next = mt->next;
242 delete mt;
243 mt = next;
244 }
245 lcContext.MsgTypes = nullptr;
246 }
247 }
248
249
250
251
252 /**
253 Customizing memory management for LowComm subsystem.
254 Currently this function supports only alloc/free of
255 buffers for messages to be sent.
256 */
257 void LC_SetMemMgrSend(DDD::DDDContext& context, AllocFunc aAllocFunc, FreeFunc aFreeFunc)
258 {
259 auto& lcContext = context.lowCommContext();
260 lcContext.SendAlloc = aAllocFunc;
261 lcContext.SendFree = aFreeFunc;
262 }
263
264
265 /**
266 Customizing memory management for LowComm subsystem.
267 Currently this function supports only alloc/free of
268 buffers for messages to be received.
269 */
270 void LC_SetMemMgrRecv(DDD::DDDContext& context, AllocFunc aAllocFunc, FreeFunc aFreeFunc)
271 {
272 auto& lcContext = context.lowCommContext();
273 lcContext.RecvAlloc = aAllocFunc;
274 lcContext.RecvFree = aFreeFunc;
275 }
276
277
278 /**
279 Set memory management for LowComm subsystem to its default state.
280 Set alloc/free of message buffers to the functions provided to
281 \lcfunk{Init}.
282 */
283 void LC_SetMemMgrDefault(DDD::DDDContext& context)
284 {
285 auto& lcContext = context.lowCommContext();
286 lcContext.SendAlloc = lcContext.DefaultAlloc;
287 lcContext.SendFree = lcContext.DefaultFree;
288 lcContext.RecvAlloc = lcContext.DefaultAlloc;
289 lcContext.RecvFree = lcContext.DefaultFree;
290 }
291
292
293 /****************************************************************************/
294
295 /*
296 auxiliary functions
297 */
298
299 static MSG_DESC *NewMsgDesc (DDD::DDDContext& context)
300 {
301 auto& lcContext = context.lowCommContext();
302 MSG_DESC *md;
303
304 if (lcContext.FreeMsgDescs != nullptr)
305 {
306 /* get item from freelist */
307 md = lcContext.FreeMsgDescs;
308 lcContext.FreeMsgDescs = lcContext.FreeMsgDescs->next;
309 }
310 else
311 {
312 /* freelist is empty */
313 md = new MSG_DESC;
314 }
315
316 return(md);
317 }
318
319
320 static void FreeMsgDesc (DDD::DDDContext& context, MSG_DESC *md)
321 {
322 auto& lcContext = context.lowCommContext();
323
324 /* sort into freelist */
325 md->next = lcContext.FreeMsgDescs;
326 lcContext.FreeMsgDescs = md;
327 }
328
329
330 /****************************************************************************/
331
332
333
334 /*
335 this function has internal access only because LowComm initiates
336 asynchronous receive calls itself.
337 */
338
339 static LC_MSGHANDLE LC_NewRecvMsg (DDD::DDDContext& context, LC_MSGTYPE mtyp, DDD_PROC source, size_t size)
340 {
341 auto& lcContext = context.lowCommContext();
342 MSG_DESC *msg = NewMsgDesc(context);
343
344 # if DebugLowComm<=6
345 Dune::dverb << "LC_NewRecvMsg(" << mtyp->name << ") source=" << source << "\n";
346 # endif
347
348 msg->msgState = MSTATE_NEW;
349 msg->msgType = mtyp;
350 msg->proc = source;
351 msg->bufferSize = size;
352
353 /* allocate chunks array */
354 msg->chunks = new CHUNK_DESC[mtyp->nComps];
355
356 /* enter message into recv queue */
357 msg->next = lcContext.RecvQueue;
358 lcContext.RecvQueue = msg;
359
360 return msg;
361 }
362
363
364
365
366 static void LC_DeleteMsg (DDD::DDDContext& context, LC_MSGHANDLE md)
367 {
368 delete[] md->chunks;
369 FreeMsgDesc(context, md);
370 }
371
372
373 static void LC_DeleteMsgBuffer (const DDD::DDDContext& context, LC_MSGHANDLE md)
374 {
375 const auto& lcContext = context.lowCommContext();
376
377 if (lcContext.SendFree != nullptr)
378 (*lcContext.SendFree)(md->buffer);
379 }
380
381
382
383
384 static void LC_MsgRecv (MSG_DESC *md)
385 {
386 int i, j;
387
388 /* get message address */
389 const ULONG *hdr = (ULONG *)md->buffer;
390
391 /* get number of chunks */
392 int n = (int)(hdr[1]);
393
394 /* magic number is hdr[0] */
395 if (hdr[0] != MAGIC_DUMMY)
396 DUNE_THROW(Dune::Exception, "invalid magic number for message from " << md->proc);
397
398 /* number of chunks must be consistent with message type */
399 if (n!=md->msgType->nComps)
400 DUNE_THROW(Dune::Exception,
401 "wrong number of chunks (got " << n << ", expected "
402 << md->msgType->nComps << ") in message from " << md->proc);
403
404 /* get chunk descriptions from message header */
405 for(j=2, i=0; i<n; i++)
406 {
407 md->chunks[i].offset = hdr[j++];
408 md->chunks[i].size = (size_t)(hdr[j++]);
409 md->chunks[i].entries = hdr[j++];
410 }
411
412 #if DebugLowComm<=2
413 Dune:dvverb << "LC_MsgRecv() from=" << md->proc << " ready\n";
414 #endif
415 }
416
417
418
419 /****************************************************************************/
420 /* */
421 /* Function: LC_PollSend */
422 /* */
423 /* Purpose: polls all message-sends one time and returns remaining */
424 /* outstanding messages. whenever a message-send has been */
425 /* completed, its message buffer is freed. */
426 /* */
427 /* Input: - */
428 /* */
429 /* Output: remaining outstanding messages */
430 /* */
431 /****************************************************************************/
432
433 static int LC_PollSend(const DDD::DDDContext& context)
434 {
435 const auto& lcContext = context.lowCommContext();
436
437 MSG_DESC *md;
438 int remaining, error;
439
440 remaining = 0;
441 for(md=lcContext.SendQueue; md != nullptr; md=md->next)
442 {
443 if (md->msgState==MSTATE_COMM)
444 {
445 error = InfoASend(context.ppifContext(), VCHAN_TO(context, md->proc), md->msgId);
446 if (error==-1)
447 DUNE_THROW(Dune::Exception, "InfoASend() failed for message to proc=" << md->proc);
448
449 if (error==1)
450 {
451 /* free message buffer */
452 LC_DeleteMsgBuffer(context, (LC_MSGHANDLE)md);
453
454 md->msgState=MSTATE_READY;
455 }
456 else
457 {
458 /* we keep this message in SendQueue */
459 remaining++;
460 }
461 }
462 }
463
464 #if DebugLowComm<=3
465 Dune::dvverb << "LC_PollSend, " << remaining << " msgs remaining\n";
466 #endif
467
468 return(remaining);
469 }
470
471
472
473 /****************************************************************************/
474 /* */
475 /* Function: LC_PollRecv */
476 /* */
477 /* Purpose: polls all message-recvs one time and returns remaining */
478 /* outstanding messages. this function doesn't free the message */
479 /* buffers. */
480 /* */
481 /* Input: - */
482 /* */
483 /* Output: remaining outstanding messages */
484 /* */
485 /****************************************************************************/
486
487 static int LC_PollRecv(const DDD::DDDContext& context)
488 {
489 const auto& lcContext = context.lowCommContext();
490
491 MSG_DESC *md;
492 int remaining, error;
493
494 remaining = 0;
495 for(md=lcContext.RecvQueue; md != nullptr; md=md->next)
496 {
497 if (md->msgState==MSTATE_COMM)
498 {
499 error = InfoARecv(context.ppifContext(), VCHAN_TO(context, md->proc), md->msgId);
500 if (error==-1)
501 DUNE_THROW(Dune::Exception,
502 "InfoARecv() failed for recv from proc=" << md->proc);
503
504 if (error==1)
505 {
506 LC_MsgRecv(md);
507
508 md->msgState=MSTATE_READY;
509 }
510 else
511 {
512 remaining++;
513 }
514 }
515 }
516
517 #if DebugLowComm<=3
518 Dune::dvverb << "LC_PollRecv, " << remaining << " msgs remaining\n";
519 #endif
520
521 return(remaining);
522 }
523
524
525 /****************************************************************************/
526 /* */
527 /* Function: LC_FreeSendQueue */
528 /* */
529 /****************************************************************************/
530
531 static void LC_FreeSendQueue (DDD::DDDContext& context)
532 {
533 auto& lcContext = context.lowCommContext();
534 MSG_DESC *md, *next=NULL;
535
536 for(md=lcContext.SendQueue; md != nullptr; md=next)
537 {
538 /* the following assertion is too picky. Freeing of
539 message queues should be possible in all msgStates. */
540 /*assert(md->msgState==MSTATE_READY);*/
541
542 next = md->next;
543 LC_DeleteMsg(context, (LC_MSGHANDLE)md);
544 }
545
546
547 lcContext.SendQueue = nullptr;
548 lcContext.nSends = 0;
549 }
550
551
552 /****************************************************************************/
553 /* */
554 /* Function: LC_FreeRecvQueue */
555 /* */
556 /****************************************************************************/
557
558 static void LC_FreeRecvQueue (DDD::DDDContext& context)
559 {
560 auto& lcContext = context.lowCommContext();
561 MSG_DESC *md, *next=NULL;
562
563 for(md=lcContext.RecvQueue; md != nullptr; md=next)
564 {
565 /* the following assertion is too picky. Freeing of
566 message queues should be possible in all msgStates. */
567 /*assert(md->msgState==MSTATE_READY);*/
568
569 next = md->next;
570 LC_DeleteMsg(context, (LC_MSGHANDLE)md);
571 }
572
573
574 lcContext.RecvQueue = nullptr;
575 lcContext.nRecvs = 0;
576 }
577
578
579
580 /****************************************************************************/
581
582
583 /* LC_MsgFreeze and LC_MsgAlloc are the two parts of LC_MsgPrepareSend(). */
584
585 /* returns size of message buffer */
586
587 size_t LC_MsgFreeze (LC_MSGHANDLE md)
588 {
589 int i, n = md->msgType->nComps;
590
591 assert(md->msgState==MSTATE_NEW);
592
593 /* compute size of header */
594 md->bufferSize = 2 * sizeof(ULONG);
595 md->bufferSize += (n * HDR_ENTRIES_PER_CHUNK * sizeof(ULONG));
596
597 /* compute size and offset for each chunk */
598 for(i=0; i<n; i++)
599 {
600 md->chunks[i].offset = md->bufferSize;
601 md->bufferSize += md->chunks[i].size;
602 }
603
604 md->msgState=MSTATE_FREEZED;
605
606 return(md->bufferSize);
607 }
608
609
610
611 int LC_MsgAlloc(DDD::DDDContext& context, LC_MSGHANDLE md)
612 {
613 auto& lcContext = context.lowCommContext();
614 ULONG *hdr;
615 int i, j, n = md->msgType->nComps;
616 int remaining=1, give_up = false;
617
618 assert(md->msgState==MSTATE_FREEZED);
619
620 /* the following code tries to allocate the message buffer.
621 if this fails, the previously started asynchronous sends are
622 polled, in order to free their message buffers. if there are
623 no remaining async-sends, we give up. */
624 do {
625 /* allocate buffer for messages */
626 md->buffer = (char *) (*lcContext.SendAlloc)(md->bufferSize);
627 if (md->buffer==NULL)
628 {
629 if (remaining==0)
630 give_up = true;
631 else
632 {
633 # if DebugLowComm<=7
634 Dune::dinfo << "LC_MsgAlloc(" << md->msgType->name
635 << ") detected low memory.\n";
636 # endif
637
638 /* couldn't get msg-buffer. try to poll previous messages. */
639 /* first, poll receives to avoid communication deadlock. */
640 LC_PollRecv(context);
641
642 /* now, try to poll sends and free their message buffers */
643 remaining = LC_PollSend(context);
644
645 # if DebugLowComm<=6
646 Dune::dverb << "LC_MsgAlloc(" << md->msgType->name
647 << ") preliminary poll, sends_left=" << remaining << "\n";
648 # endif
649 }
650 }
651 } while (md->buffer==NULL && !give_up);
652
653 if (give_up)
654 {
655 # if DebugLowComm<=7
656 Dune::dinfo << "LC_MsgAlloc(" << md->msgType->name << ") giving up, no memory.\n";
657 # endif
658 return(false);
659 }
660
661
662 /* enter control data into message header */
663 hdr = (ULONG *)md->buffer;
664 j=0;
665 hdr[j++] = MAGIC_DUMMY; /* magic number */
666 hdr[j++] = n;
667
668 /* enter chunk descriptions into message header */
669 for(i=0; i<n; i++)
670 {
671 hdr[j++] = md->chunks[i].offset;
672 hdr[j++] = md->chunks[i].size;
673 hdr[j++] = md->chunks[i].entries;
674 }
675
676 md->msgState=MSTATE_ALLOCATED;
677
678 return(true);
679 }
680
681
682
683
684 /*
685 allocation of receive message buffers.
686 NOTE: one big memory block is allocated and used for all
687 message buffers.
688 */
689 static RETCODE LC_PrepareRecv(DDD::DDDContext& context)
690 {
691 auto& lcContext = context.lowCommContext();
692 MSG_DESC *md;
693 size_t sumSize;
694 char *buffer;
695 int error;
696
697 /* compute sum of message buffer sizes */
698 for(sumSize=0, md=lcContext.RecvQueue; md != nullptr; md=md->next)
699 {
700 assert(md->msgState==MSTATE_NEW);
701 sumSize += md->bufferSize;
702 }
703
704
705 /* allocate buffer for messages */
706 lcContext.theRecvBuffer = (char *) (*lcContext.RecvAlloc)(sumSize);
707 if (lcContext.theRecvBuffer == nullptr)
708 {
709 Dune::dwarn << "Out of memory in LC_PrepareRecv "
710 << "(size of message buffer: " << sumSize << ")";
711 RET_ON_ERROR;
712 }
713
714
715 /* initiate receive calls */
716 buffer = lcContext.theRecvBuffer;
717 for(md=lcContext.RecvQueue; md != nullptr; md=md->next)
718 {
719 md->buffer = buffer;
720 buffer += md->bufferSize;
721
722 md->msgId = RecvASync(context.ppifContext(), VCHAN_TO(context, md->proc),
723 md->buffer, md->bufferSize, &error);
724
725 md->msgState=MSTATE_COMM;
726 }
727
728 RET_ON_OK;
729 }
730
731
732 /****************************************************************************/
733
734 /*
735 MSG_TYPE definition functions
736 */
737
738
739 /****************************************************************************/
740 /* */
741 /* Function: LC_NewMsgType */
742 /* */
743 /****************************************************************************/
744
745 /**
746 Declares new message-type.
747 Before messages may be sent and received with the LowComm
748 subsystem, at least one {\em message-type} must be defined by
749 a global call to this function. Subsequently, calls to
750 \lcfunk{NewMsgTable} and \lcfunk{NewMsgChunk} can be used
751 in order to define the structure of the new message-type.
752
753 Each message-type in the LowComm subsystem consists of a set
754 of {\em message-components}. Possible message-components are:
755 {\em tables} (with entries of equal size) and raw {\em data chunks}.
756 The set of message-components has the same structure for
757 all messages of the same type, but the number of table entries
758 and the size of the data chunks differ from message to message.
759
760 @return identifier of new message-type
761 @param aName name of message-type. This string is used for debugging
762 and logging output.
763 */
764
765 LC_MSGTYPE LC_NewMsgType(DDD::DDDContext& context, const char *aName)
766 {
767 auto& lcContext = context.lowCommContext();
768 MSG_TYPE *mt;
769
770 mt = new MSG_TYPE;
771 mt->name = aName;
772 mt->nComps = 0;
773
774 /* insert into linked list of message types */
775 mt->next = lcContext.MsgTypes;
776 lcContext.MsgTypes = mt;
777
778 return mt;
779 }
780
781
782
783 /****************************************************************************/
784 /* */
785 /* Function: LC_NewMsgChunk */
786 /* */
787 /****************************************************************************/
788
789 /**
790 Add data chunk to current set of a message-type's message-components.
791 This function is called after a previous call to \lcfunk{NewMsgType}
792 in order to add a new message-component to the message-type.
793 The component added by this function is a chunk of raw data.
794 The size of the chunk is not specified here, use \lcfunk{SetChunkSize}
795 for specifying the data chunk size for a given (concrete) message.
796
797 See \lcfunk{NewMsgTable} for adding message-tables, which are
798 a different kind of message-component.
799
800 @return identifier of new message-component
801 @param aName name of new message component
802 @param mtyp previously declared message-type
803 */
804
805 LC_MSGCOMP LC_NewMsgChunk (const char *aName, LC_MSGTYPE mtyp)
806 {
807 LC_MSGCOMP id = mtyp->nComps++;
808
809 if (id>=MAX_COMPONENTS)
810 DUNE_THROW(Dune::Exception,
811 "too many message components (max. " << MAX_COMPONENTS << ")");
812
813 mtyp->comp[id].type = CT_CHUNK;
814 mtyp->comp[id].name = aName;
815
816 return(id);
817 }
818
819
820
821
822 /****************************************************************************/
823 /* */
824 /* Function: LC_NewMsgTable */
825 /* */
826 /****************************************************************************/
827
828 /**
829 Add table to current set of a message-type's message-components.
830 This function is called after a previous call to \lcfunk{NewMsgType}
831 in order to add a new message-component to the message-type.
832 The component added by this function is a table of data, where
833 each table entry has the same size.
834 The overall size of the whole table is not specified here, but only
835 the size for one table entry. Use \lcfunk{SetTableSize} for setting
836 the number of reserved table entries in a given (concrete) message;
837 use \lcfunk{SetTableLen} in order to specify the number of valid
838 entries in a given message.
839
840 See \lcfunk{NewMsgChunk} for adding data chunks, which are
841 a different kind of message-component.
842
843 @return identifier of new message-component
844 @param aName name of new message component
845 @param mtyp previously declared message-type
846 @param aSize size of each table entry (in byte)
847 */
848
849 LC_MSGCOMP LC_NewMsgTable (const char *aName, LC_MSGTYPE mtyp, size_t aSize)
850 {
851 LC_MSGCOMP id = mtyp->nComps++;
852
853 if (id>=MAX_COMPONENTS)
854 DUNE_THROW(Dune::Exception,
855 "too many message components (max. " << MAX_COMPONENTS << ")");
856
857 mtyp->comp[id].type = CT_TABLE;
858 mtyp->comp[id].entry_size = aSize;
859 mtyp->comp[id].name = aName;
860
861 return(id);
862 }
863
864
865
866 /****************************************************************************/
867
868
869
870 /****************************************************************************/
871 /* */
872 /* Function: LC_NewSendMsg */
873 /* */
874 /****************************************************************************/
875
876 /**
877 Create new message on sending processor.
878 This function creates a new message handle on the sending processor and
879 links it into the LowComm send-queue. The message has a given message-type
880 and a given destination processor. Before the message is actually sent
881 (by calling \lcfunk{MsgSend}), the sizes of the message's components
882 must be set (\lcfunk{SetTableSize}, \lcfunk{SetChunkSize}) and the message
883 buffer must be prepared (via \lcfunk{MsgPrepareSend}). After that,
884 the message's tables and chunks can be filled with data and the message
885 sending process can be initiated by \lcfunk{MsgSend}.
886
887 @return identifier of new message
888 @param mtyp message-type for new message
889 @param aDest destination processor of new message
890 */
891
892 LC_MSGHANDLE LC_NewSendMsg(DDD::DDDContext& context, LC_MSGTYPE mtyp, DDD_PROC aDest)
893 {
894 auto& lcContext = context.lowCommContext();
895
896 MSG_DESC *msg = NewMsgDesc(context);
897
898 # if DebugLowComm<=6
899 Dune::dverb << "LC_NewSendMsg(" << mtyp->name << ") dest="
900 << aDest << " nSends=" << (lcContext.nSends+1) << "\n";
901 # endif
902
903
904 msg->msgState = MSTATE_NEW;
905 msg->msgType = mtyp;
906 msg->proc = aDest;
907 msg->bufferSize = 0;
908
909 /* allocate chunks array */
910 msg->chunks = new CHUNK_DESC[mtyp->nComps];
911
912 /* enter message into send queue */
913 msg->next = lcContext.SendQueue;
914 lcContext.SendQueue = msg;
915 lcContext.nSends++;
916
917 return msg;
918 }
919
920
921
922
923 void LC_SetChunkSize (LC_MSGHANDLE md, LC_MSGCOMP id, size_t size)
924 {
925 assert(md->msgState==MSTATE_NEW);
926 assert(id < md->msgType->nComps);
927
928 md->chunks[id].size = size;
929 md->chunks[id].entries = 1;
930 }
931
932
933 void LC_SetTableSize (LC_MSGHANDLE md, LC_MSGCOMP id, ULONG entries)
934 {
935 assert(md->msgState==MSTATE_NEW);
936 assert(id < md->msgType->nComps);
937
938 md->chunks[id].size = ((int)entries) * md->msgType->comp[id].entry_size;
939 md->chunks[id].entries = entries;
940 }
941
942
943
944 /****************************************************************************/
945 /* */
946 /* Function: LC_MsgPrepareSend */
947 /* */
948 /****************************************************************************/
949
950
951 /* returns size of message buffer */
952
953 size_t LC_MsgPrepareSend (DDD::DDDContext& context, LC_MSGHANDLE msg)
954 {
955 size_t size = LC_MsgFreeze(msg);
956 if (! LC_MsgAlloc(context, msg))
957 throw std::bad_alloc();
958
959 return(size);
960 }
961
962
963
964
965 DDD_PROC LC_MsgGetProc (const LC_MSGHANDLE md)
966 {
967 return md->proc;
968 }
969
970
971 void *LC_GetPtr (LC_MSGHANDLE md, LC_MSGCOMP id)
972 {
973 return ((void *)(((char *)md->buffer) + md->chunks[id].offset));
974 }
975
976
977 void LC_SetTableLen (LC_MSGHANDLE md, LC_MSGCOMP id, ULONG n)
978 {
979 ULONG *hdr = (ULONG *)md->buffer;
980
981 hdr[HDR_ENTRIES_PER_CHUNK*id+4] = n;
982 md->chunks[id].entries = n;
983 }
984
985
986 ULONG LC_GetTableLen (LC_MSGHANDLE md, LC_MSGCOMP id)
987 {
988 return((ULONG)md->chunks[id].entries);
989 }
990
991
992 void LC_MsgSend(const DDD::DDDContext& context, LC_MSGHANDLE md)
993 {
994 int error;
995
996 assert(md->msgState==MSTATE_ALLOCATED);
997
998 /* initiate asynchronous send */
999 md->msgId = SendASync(context.ppifContext(), VCHAN_TO(context, md->proc),
1000 md->buffer, md->bufferSize, &error);
1001
1002 md->msgState=MSTATE_COMM;
1003 }
1004
1005
1006 size_t LC_GetBufferSize (const LC_MSGHANDLE md)
1007 {
1008 return(md->bufferSize);
1009 }
1010
1011
1012 /****************************************************************************/
1013
1014
1015 /****************************************************************************/
1016 /* */
1017 /* Function: LC_Connect */
1018 /* */
1019 /****************************************************************************/
1020
1021 int LC_Connect(DDD::DDDContext& context, LC_MSGTYPE mtyp)
1022 {
1023 auto& lcContext = context.lowCommContext();
1024
1025 DDD_PROC *partners = DDD_ProcArray(context);
1026 NOTIFY_DESC *msgs = DDD_NotifyBegin(context, lcContext.nSends);
1027 MSG_DESC *md;
1028 int i, p;
1029
1030 const auto procs = context.procs();
1031
1032 if (lcContext.nSends<0 || lcContext.nSends>procs-1)
1033 DUNE_THROW(Dune::Exception,
1034 "cannot send " << lcContext.nSends << "messages "
1035 "(must be less than " << (procs-1) << ")");
1036
1037 # if DebugLowComm<=9
1038 Dune::dinfo << "LC_Connect(" << mtyp->name
1039 << ") nSends=" << lcContext.nSends << " ...\n";
1040 # endif
1041
1042
1043
1044 /* fill notify array */
1045 for(i=0, p=0, md=lcContext.SendQueue; md != nullptr; i++, md=md->next)
1046 {
1047 msgs[i].proc = md->proc;
1048 msgs[i].size = md->bufferSize;
1049
1050 /* enhance list of communication partners (destinations) */
1051 partners[p++] = md->proc;
1052
1053 }
1054
1055
1056 /* inform message receivers */
1057 lcContext.nRecvs = DDD_Notify(context);
1058 if (lcContext.nRecvs<0)
1059 {
1060 /* some processor raised an exception */
1061 Dune::dwarn << "Notify() raised exception #"
1062 << (-lcContext.nRecvs) << " in LC_Connect()\n";
1063
1064 /* automatically call LC_Cleanup() */
1065 DDD_NotifyEnd(context);
1066 LC_Cleanup(context);
1067
1068 return lcContext.nRecvs;
1069 }
1070
1071
1072 if (lcContext.nRecvs>procs-1)
1073 {
1074 Dune::dwarn << "cannot receive " << lcContext.nRecvs
1075 << " messages (must be less than " << (procs-1) << ")\n";
1076 DDD_NotifyEnd(context);
1077 return(EXCEPTION_LOWCOMM_CONNECT);
1078 }
1079
1080
1081
1082 # if DebugLowComm<=7
1083 Dune::dinfo << "LC_Connect() nSends=" << lcContext.nSends
1084 << " nRecvs=" << lcContext.nRecvs << "\n";
1085 # endif
1086
1087
1088 /* create array of receive message handles */
1089 if (lcContext.nRecvs>0)
1090 lcContext.theRecvArray = new LC_MSGHANDLE[lcContext.nRecvs];
1091
1092
1093 /* create recv messages from notify array */
1094 for(i=0; i < lcContext.nRecvs; i++)
1095 {
1096 /* create recv message handle and store it in MSGHANDLE array */
1097 lcContext.theRecvArray[i] = LC_NewRecvMsg(context, mtyp, msgs[i].proc, msgs[i].size);
1098
1099 /* enhance list of communication partners (sources) */
1100 partners[p++] = msgs[i].proc;
1101
1102 }
1103
1104
1105 DDD_NotifyEnd(context);
1106
1107
1108 /* get necessary connections to comm-partners */
1109 if (p>0)
1110 {
1111 if (! IS_OK(DDD_GetChannels(context, lcContext.nRecvs+lcContext.nSends)))
1112 {
1113 DDD_PrintError('E', 6620, "couldn't get channels in LC_Connect()");
1114 return(EXCEPTION_LOWCOMM_CONNECT);
1115 }
1116 }
1117
1118
1119 # if DebugLowComm<=5
1120 DDD_DisplayTopo(context);
1121 # endif
1122
1123
1124 if (lcContext.nRecvs>0)
1125 {
1126 if (! IS_OK(LC_PrepareRecv(context)))
1127 return(EXCEPTION_LOWCOMM_CONNECT);
1128 }
1129
1130
1131 # if DebugLowComm<=9
1132 Dune::dinfo << "LC_Connect() ready\n";
1133 # endif
1134
1135 return lcContext.nRecvs;
1136 }
1137
1138
1139
1140 /****************************************************************************/
1141 /* */
1142 /* Function: LC_Abort */
1143 /* */
1144 /****************************************************************************/
1145
1146 int LC_Abort(DDD::DDDContext& context, int exception)
1147 {
1148 int retException;
1149
1150 if (exception>EXCEPTION_LOWCOMM_USER)
1151 DUNE_THROW(Dune::Exception,
1152 "exception must be <= EXCEPTION_LOWCOMM_USER");
1153
1154 DDD_NotifyBegin(context, exception);
1155
1156 # if DebugLowComm<=9
1157 Dune::dwarn << "LC_Abort() exception=" << exception << " ...\n";
1158 # endif
1159
1160
1161 /* inform message receivers */
1162 retException = DDD_Notify(context);
1163
1164 DDD_NotifyEnd(context);
1165
1166
1167 # if DebugLowComm<=9
1168 Dune::dwarn << "LC_Abort() ready, exception=" << retException << "\n";
1169 # endif
1170
1171
1172 /* automatically call LC_Cleanup() */
1173 LC_Cleanup(context);
1174
1175 return(retException);
1176 }
1177
1178
1179
1180 /****************************************************************************/
1181 /* */
1182 /* Function: LC_Communicate */
1183 /* */
1184 /****************************************************************************/
1185
1186 LC_MSGHANDLE *LC_Communicate(const DDD::DDDContext& context)
1187 {
1188 auto& lcContext = context.lowCommContext();
1189
1190 # if DebugLowComm<=9
1191 Dune::dinfo << "LC_Communicate() ...\n";
1192 # endif
1193
1194
1195 /* poll asynchronous send and receives */
1196 int leftSend = lcContext.nSends;
1197 int leftRecv = lcContext.nRecvs;
1198 do {
1199 if (leftRecv>0) leftRecv = LC_PollRecv(context);
1200 if (leftSend>0) leftSend = LC_PollSend(context);
1201 } while (leftRecv>0 || leftSend>0);
1202
1203
1204 # if DebugLowComm<=9
1205 Dune::dinfo << "LC_Communicate() ready\n";
1206 # endif
1207
1208 return lcContext.theRecvArray;
1209 }
1210
1211
1212 /****************************************************************************/
1213 /* */
1214 /* Function: LC_Cleanup */
1215 /* */
1216 /****************************************************************************/
1217
1218 void LC_Cleanup(DDD::DDDContext& context)
1219 {
1220 auto& lcContext = context.lowCommContext();
1221
1222 # if DebugLowComm<=9
1223 Dune::dinfo << "LC_Cleanup() ...\n";
1224 # endif
1225
1226 if (lcContext.nRecvs>0)
1227 {
1228 if (lcContext.RecvFree != nullptr)
1229 (lcContext.RecvFree)(lcContext.theRecvBuffer);
1230
1231 lcContext.theRecvBuffer = nullptr;
1232 }
1233
1234 if (lcContext.theRecvArray != nullptr)
1235 {
1236 delete[] lcContext.theRecvArray;
1237 lcContext.theRecvArray = nullptr;
1238 }
1239
1240 /* free recv queue */
1241 LC_FreeRecvQueue(context);
1242
1243 /* free send queue */
1244 LC_FreeSendQueue(context);
1245
1246
1247 # if DebugLowComm<=9
1248 Dune::dinfo << "LC_Cleanup() ready\n";
1249 # endif
1250 }
1251
1252
1253 /****************************************************************************/
1254
1255 #define LC_COLWIDTH 10
1256
1257 static const char* LC_DefaultName = "<?>";
1258
1259 /* construct name or default name */
1260 static const char* LC_Name(const char* name)
1261 {
1262 return name ? name : LC_DefaultName;
1263 }
1264
1265 static void LC_PrintMsgList (MSG_DESC *list)
1266 {
1267 using std::setw;
1268
1269 std::ostream& out = std::cout;
1270 MSG_DESC *md;
1271 MSG_TYPE *last_mt=NULL;
1272 size_t sum, comp_size[MAX_COMPONENTS];
1273 int i;
1274
1275 for(md=list; md != nullptr; md=md->next)
1276 {
1277 MSG_TYPE *mt = md->msgType;
1278
1279 if (mt!=last_mt)
1280 {
1281 /* msg-type changes, print new header */
1282
1283 /* first, close part of msg-list with summary */
1284 if (last_mt!=NULL)
1285 {
1286 out << " = |";
1287 sum = 0;
1288 for(i=0; i<last_mt->nComps; i++)
1289 {
1290 out << setw(9) << comp_size[i];
1291 sum += comp_size[i]; /* horizontal sum */
1292 }
1293 out << setw(9) << sum << "\n";
1294 }
1295
1296 /* then, construct header */
1297 {
1298 std::string name = LC_Name(mt->name);
1299 out << setw(9) << name.substr(0, 9) << " |";
1300 }
1301 for(i=0; i<mt->nComps; i++)
1302 {
1303 if (mt->comp[i].name!=NULL) {
1304 std::string name = LC_Name(mt->comp[i].name);
1305 out << setw(9) << name.substr(0, 9);
1306 }
1307 else
1308 out << setw(9) << i;
1309
1310 comp_size[i] = 0;
1311 }
1312 out << " =\n";
1313 last_mt = mt;
1314 }
1315
1316 /* construct info about message components */
1317 out << setw(9) << md->proc << " |";
1318 sum = 0;
1319 for(i=0; i<mt->nComps; i++)
1320 {
1321 size_t s = md->chunks[i].size;
1322
1323 out << setw(9) << s;
1324
1325 sum += s; /* horizontal sum */
1326 comp_size[i] += s; /* vertical sum */
1327 }
1328 out << setw(9) << sum << "\n";
1329 }
1330
1331 /* close last part of msg-list with summary */
1332 if (last_mt!=NULL)
1333 {
1334 out << " = |";
1335 sum = 0;
1336 for(i=0; i<last_mt->nComps; i++)
1337 {
1338 out << setw(9) << comp_size[i];
1339 sum += comp_size[i]; /* horizontal sum */
1340 }
1341 out << setw(9) << sum << "\n";
1342 }
1343
1344 }
1345
1346
1347 void LC_PrintSendMsgs(const DDD::DDDContext& context)
1348 {
1349 LC_PrintMsgList(context.lowCommContext().SendQueue);
1350 }
1351
1352 void LC_PrintRecvMsgs(const DDD::DDDContext& context)
1353 {
1354 LC_PrintMsgList(context.lowCommContext().RecvQueue);
1355 }
1356
1357
1358 /****************************************************************************/
1359
1360 } /* namespace DDD */