| 1 |
// SPDX-FileCopyrightText: Copyright © DUNE Project contributors, see file LICENSE.md in module root |
| 2 |
// SPDX-License-Identifier: LGPL-2.1-or-later |
| 3 |
// -*- tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 2 -*- |
| 4 |
// vi: set et ts=4 sw=2 sts=2: |
| 5 |
/****************************************************************************/ |
| 6 |
/* */ |
| 7 |
/* File: topo.c */ |
| 8 |
/* */ |
| 9 |
/* Purpose: maintains communication structure for data-dependent */ |
| 10 |
/* communication topology */ |
| 11 |
/* */ |
| 12 |
/* Author: Klaus Birken */ |
| 13 |
/* Rechenzentrum Uni Stuttgart */ |
| 14 |
/* Universitaet Stuttgart */ |
| 15 |
/* Allmandring 30 */ |
| 16 |
/* 70550 Stuttgart */ |
| 17 |
/* internet: [email protected] */ |
| 18 |
/* */ |
| 19 |
/* History: 93/11/30 kb begin */ |
| 20 |
/* 95/10/05 kb added channel disconnection to TopoExit */ |
| 21 |
/* */ |
| 22 |
/* Remarks: */ |
| 23 |
/* */ |
| 24 |
/****************************************************************************/ |
| 25 |
|
| 26 |
/****************************************************************************/ |
| 27 |
/* */ |
| 28 |
/* include files */ |
| 29 |
/* system include files */ |
| 30 |
/* application include files */ |
| 31 |
/* */ |
| 32 |
/****************************************************************************/ |
| 33 |
|
| 34 |
/* standard C library */ |
| 35 |
#include <config.h> |
| 36 |
#include <cstdlib> |
| 37 |
#include <cstdio> |
| 38 |
#include <cstring> |
| 39 |
|
| 40 |
#include <iomanip> |
| 41 |
#include <iostream> |
| 42 |
#include <vector> |
| 43 |
|
| 44 |
#include <dune/common/stdstreams.hh> |
| 45 |
|
| 46 |
#include <dune/uggrid/parallel/ddd/dddcontext.hh> |
| 47 |
|
| 48 |
#include <dune/uggrid/low/namespace.h> |
| 49 |
|
| 50 |
#include <dune/uggrid/parallel/ddd/dddi.h> |
| 51 |
#include <dune/uggrid/parallel/ddd/include/memmgr.h> |
| 52 |
|
| 53 |
USING_UG_NAMESPACES |
| 54 |
|
| 55 |
/* PPIF namespace: */ |
| 56 |
using namespace PPIF; |
| 57 |
|
| 58 |
namespace DDD { |
| 59 |
|
| 60 |
/****************************************************************************/ |
| 61 |
/* */ |
| 62 |
/* routines */ |
| 63 |
/* */ |
| 64 |
/****************************************************************************/ |
| 65 |
|
| 66 |
|
| 67 |
/* TODO memory usage is O(P) in current implementation! */ |
| 68 |
|
| 69 |
void ddd_TopoInit(DDD::DDDContext& context) |
| 70 |
{ |
| 71 |
auto& ctx = context.topoContext(); |
| 72 |
const auto procs = context.procs(); |
| 73 |
|
| 74 |
/* get one channel pointer for each partner */ |
| 75 |
ctx.theTopology.assign(procs, nullptr); |
| 76 |
|
| 77 |
/* get proc array with maxsize = 2 * number of procs */ |
| 78 |
ctx.theProcArray.resize(2 * procs); |
| 79 |
} |
| 80 |
|
| 81 |
|
| 82 |
void ddd_TopoExit(DDD::DDDContext& context) |
| 83 |
{ |
| 84 |
auto& ctx = context.topoContext(); |
| 85 |
|
| 86 |
ctx.theProcArray.clear(); |
| 87 |
|
| 88 |
/* disconnect channels */ |
| 89 |
for (const auto ch : ctx.theTopology) |
| 90 |
{ |
| 91 |
if (ch != nullptr) |
| 92 |
{ |
| 93 |
DiscASync(context.ppifContext(), ch); |
| 94 |
while (InfoADisc(context.ppifContext(), ch)!=1) |
| 95 |
; |
| 96 |
} |
| 97 |
} |
| 98 |
|
| 99 |
ctx.theTopology.clear(); |
| 100 |
} |
| 101 |
|
| 102 |
|
| 103 |
/****************************************************************************/ |
| 104 |
|
| 105 |
|
| 106 |
DDD_PROC* DDD_ProcArray(DDD::DDDContext& context) |
| 107 |
{ |
| 108 |
return context.topoContext().theProcArray.data(); |
| 109 |
} |
| 110 |
|
| 111 |
|
| 112 |
RETCODE DDD_GetChannels(DDD::DDDContext& context, int nPartners) |
| 113 |
{ |
| 114 |
auto& ctx = context.topoContext(); |
| 115 |
int i, nConn; |
| 116 |
|
| 117 |
if (nPartners > 2*(context.procs()-1)) |
| 118 |
{ |
| 119 |
DDD_PrintError('E', 1520, "topology error in DDD_GetChannels"); |
| 120 |
RET_ON_ERROR; |
| 121 |
} |
| 122 |
|
| 123 |
std::vector<bool> theProcFlags(nPartners); |
| 124 |
|
| 125 |
nConn = 0; |
| 126 |
for(i=0; i<nPartners; i++) |
| 127 |
{ |
| 128 |
if (ctx.theTopology[ctx.theProcArray[i]] == nullptr) |
| 129 |
{ |
| 130 |
VChannelPtr vc = ConnASync(context.ppifContext(), ctx.theProcArray[i], VC_TOPO); |
| 131 |
|
| 132 |
if (vc==NULL) |
| 133 |
{ |
| 134 |
Dune::dwarn << "DDD_GetChannels: can't connect to proc=" |
| 135 |
<< ctx.theProcArray[i] << "\n"; |
| 136 |
RET_ON_ERROR; |
| 137 |
} |
| 138 |
|
| 139 |
ctx.theTopology[ctx.theProcArray[i]] = vc; |
| 140 |
nConn++; |
| 141 |
|
| 142 |
theProcFlags[i] = true; |
| 143 |
} |
| 144 |
else |
| 145 |
{ |
| 146 |
theProcFlags[i] = false; |
| 147 |
} |
| 148 |
} |
| 149 |
|
| 150 |
|
| 151 |
while (nConn>0) |
| 152 |
{ |
| 153 |
for(i=0; i<nPartners; i++) |
| 154 |
{ |
| 155 |
if (theProcFlags[i]) |
| 156 |
{ |
| 157 |
int ret = InfoAConn(context.ppifContext(), ctx.theTopology[ctx.theProcArray[i]]); |
| 158 |
if (ret==-1) |
| 159 |
{ |
| 160 |
Dune::dwarn << "DDD_GetChannels: InfoAConn() failed for connect to proc=" |
| 161 |
<< ctx.theProcArray[i] << "\n"; |
| 162 |
RET_ON_ERROR; |
| 163 |
} |
| 164 |
|
| 165 |
|
| 166 |
if (ret==1) |
| 167 |
{ |
| 168 |
theProcFlags[i] = false; |
| 169 |
nConn--; |
| 170 |
} |
| 171 |
} |
| 172 |
} |
| 173 |
} |
| 174 |
/* TODO free unused channels? */ |
| 175 |
|
| 176 |
RET_ON_OK; |
| 177 |
} |
| 178 |
|
| 179 |
|
| 180 |
void DDD_DisplayTopo (const DDD::DDDContext& context) |
| 181 |
{ |
| 182 |
using std::setw; |
| 183 |
|
| 184 |
std::ostream& out = std::cout; |
| 185 |
const auto& ctx = context.topoContext(); |
| 186 |
const auto me = context.me(); |
| 187 |
const auto procs = context.procs(); |
| 188 |
|
| 189 |
DDD_SyncAll(context); |
| 190 |
|
| 191 |
if (me == 0) { |
| 192 |
out << " "; |
| 193 |
for(int p=0; p < procs; ++p) |
| 194 |
out << setw(2) << p; |
| 195 |
out << std::endl; |
| 196 |
} |
| 197 |
|
| 198 |
for(int p = 0; p < procs; ++p) |
| 199 |
{ |
| 200 |
Synchronize(context.ppifContext()); |
| 201 |
if (p == me) { |
| 202 |
out << setw(4) << me << ": "; |
| 203 |
for(int i = 0; i < procs; ++i) { |
| 204 |
if (ctx.theTopology[i]!=NULL) |
| 205 |
out << "<>"; |
| 206 |
else { |
| 207 |
if (i==p) |
| 208 |
out << "--"; |
| 209 |
else |
| 210 |
out << " "; |
| 211 |
} |
| 212 |
} |
| 213 |
out << std::endl; |
| 214 |
} |
| 215 |
} |
| 216 |
|
| 217 |
DDD_SyncAll(context); |
| 218 |
} |
| 219 |
|
| 220 |
|
| 221 |
/****************************************************************************/ |
| 222 |
|
| 223 |
} /* namespace DDD */ |