Debian Codegraph @

src/atlas/parallel/mpi/mpi.cc

raw
1 /*
2 * (C) Copyright 2013 ECMWF.
3 *
4 * This software is licensed under the terms of the Apache Licence Version 2.0
5 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
6 * In applying this licence, ECMWF does not waive the privileges and immunities
7 * granted to it by virtue of its status as an intergovernmental organisation
8 * nor does it submit to any jurisdiction.
9 */
10
11 #include "atlas/parallel/mpi/mpi.h"
12
13 #include <functional>
14 #include <memory>
15 #include <optional>
16 #include <sstream>
17 #include <stack>
18 #include <vector>
19
20 #include "atlas/runtime/Log.h"
21 #include "atlas/runtime/Exception.h"
22
23 namespace atlas::mpi {
24
25 void finalize() {
26 finalise();
27 }
28 void finalise() {
29 Log::debug() << "atlas::mpi::finalize() --> Finalizing MPI" << std::endl;
30 eckit::mpi::finaliseAllComms();
31 }
32
33 namespace {
34 const Comm& comm_world() {
35 static const Comm& world = atlas::mpi::comm("world");
36 return world;
37 }
38 const Comm& comm_self() {
39 static const Comm& self = atlas::mpi::comm("self");
40 return self;
41 }
42 int comm_world_int() {
43 static int world = comm_world().communicator();
44 return world;
45 }
46 int comm_self_int() {
47 static int self = comm_self().communicator();
48 return self;
49 }
50
51 std::string comm_name_from_communicator( int communicator ) {
52 return "int." + std::to_string(communicator);
53 }
54
55 std::optional<std::reference_wrapper<const Comm>> lookup_comm( int communicator ) {
56 // First check if the communicator corresponds to the well-known comm_world or comm_self, or the current default comm
57 // Then fall back to a more expensive lookup in the registered communicators, if any
58 if ( communicator == comm_world_int() ) {
59 return comm_world();
60 }
61 else if ( communicator == comm_self_int() ) {
62 return comm_self();
63 }
64 else {
65 const auto& default_comm = comm();
66 if ( default_comm.communicator() == communicator ) {
67 return default_comm;
68 }
69 else {
70 for( std::string name : eckit::mpi::listComms() ) {
71 const auto& comm = atlas::mpi::comm(name);
72 if ( comm.communicator() == communicator ) {
73 return comm;
74 }
75 }
76 }
77 }
78 return std::nullopt;
79 }
80
81 class CommScope {
82 public:
83 CommScope() {
84 Log::debug() << "atlas::mpi::scope::push()" << std::endl;
85 previous_comm_name_ = mpi::comm().name();
86 }
87 CommScope(std::string_view name) {
88 Log::debug() << "atlas::mpi::scope::push(" << name << ")" << std::endl;
89 previous_comm_name_ = mpi::comm().name();
90 eckit::mpi::setCommDefault(name.data());
91 }
92 CommScope(const Comm& comm) {
93 Log::debug() << "atlas::mpi::scope::push(" << comm.name() << ")" << std::endl;
94 previous_comm_name_ = mpi::comm().name();
95 eckit::mpi::setCommDefault(comm.name().data());
96 }
97 CommScope(int communicator) {
98 previous_comm_name_ = mpi::comm().name();
99 auto comm_opt = lookup_comm(communicator);
100 if (comm_opt) {
101 Log::debug() << "atlas::mpi::scope::push(" << communicator << ") : found existing comm " << comm_opt->get().name() << std::endl;
102 eckit::mpi::setCommDefault(comm_opt->get().name().data());
103 }
104 else {
105 new_registered_comm_name_ = comm_name_from_communicator(communicator);
106 Log::debug() << "atlas::mpi::scope::push(" << communicator << ") : registered new comm " << new_registered_comm_name_ << std::endl;
107 register_comm(new_registered_comm_name_, communicator);
108 eckit::mpi::setCommDefault(new_registered_comm_name_.c_str());
109 }
110 }
111 ~CommScope() {
112 Log::debug() << "atlas::mpi::scope::pop() : restored to " << previous_comm_name_;
113 eckit::mpi::setCommDefault(previous_comm_name_.c_str());
114 if (not new_registered_comm_name_.empty()) {
115 Log::debug() << " and unregistered comm " << new_registered_comm_name_;
116 unregister_comm(new_registered_comm_name_);
117 }
118 Log::debug() << '\n';
119 }
120 private:
121 std::string previous_comm_name_;
122 std::string new_registered_comm_name_;
123 };
124
125 class ScopeStack {
126 using value_type = std::unique_ptr<CommScope>;
127 using container_type = std::vector<value_type>;
128 using stack_type = std::stack<value_type, container_type>;
129 public:
130 void push() {
131 stack_.push(std::make_unique<CommScope>());
132 }
133 void push(std::string_view name) {
134 stack_.push(std::make_unique<CommScope>(name));
135 }
136 void push(const Comm& comm) {
137 stack_.push(std::make_unique<CommScope>(comm));
138 }
139 void push(int communicator) {
140 stack_.push(std::make_unique<CommScope>(communicator));
141 }
142 void pop() {
143 if( !stack_.empty() ) {
144 stack_.pop();
145 }
146 }
147 static ScopeStack& instance() {
148 static ScopeStack instance;
149 return instance;
150 }
151
152 private:
153 ScopeStack() {
154 container_type underlying;
155 underlying.reserve(64);
156 stack_ = stack_type(std::move(underlying));
157 }
158 private:
159 stack_type stack_;
160 };
161 }
162
163 void scope::push() {
164 ScopeStack::instance().push();
165 }
166
167 void scope::push(std::string_view name) {
168 ScopeStack::instance().push(name);
169 }
170
171 void scope::push(const Comm& comm) {
172 ScopeStack::instance().push(comm);
173 }
174
175 void scope::push(int communicator) {
176 ScopeStack::instance().push(communicator);
177 }
178
179 void scope::pop() {
180 ScopeStack::instance().pop();
181 }
182
183 bool has_comm( std::string_view name ) {
184 return eckit::mpi::hasComm(name.data());
185 }
186
187 const Comm& comm(int communicator) {
188 auto comm_opt = lookup_comm( communicator );
189 if ( comm_opt ) {
190 return comm_opt->get();
191 }
192 throw_Exception("No MPI communicator found with integer value " + std::to_string(communicator), Here());
193 }
194
195 const Comm& register_comm(std::string_view name, int communicator) {
196 eckit::mpi::addComm(name.data(), communicator);
197 return comm(name);
198 }
199
200 void unregister_comm(std::string_view name) {
201 #if ATLAS_ECKIT_VERSION_AT_LEAST(2, 0, 0)
202 eckit::mpi::unregisterComm(name.data());
203 #else
204 Log::warning() << "atlas::mpi::unregister_comm is a no-op with eckit versions prior to 2.0.0" << std::endl;
205 #endif
206 }
207
208
209 } // namespace atlas::mpi