Debian Codegraph @

packages/stk/stk_unit_tests/stk_util/parallel/UnitTestMPICommKey.cpp

raw
1 // Copyright 2002 - 2008, 2010, 2011 National Technology Engineering
2 // Solutions of Sandia, LLC (NTESS). Under the terms of Contract
3 // DE-NA0003525 with NTESS, the U.S. Government retains certain rights
4 // in this software.
5 //
6 // Redistribution and use in source and binary forms, with or without
7 // modification, are permitted provided that the following conditions are
8 // met:
9 //
10 // * Redistributions of source code must retain the above copyright
11 // notice, this list of conditions and the following disclaimer.
12 //
13 // * Redistributions in binary form must reproduce the above
14 // copyright notice, this list of conditions and the following
15 // disclaimer in the documentation and/or other materials provided
16 // with the distribution.
17 //
18 // * Neither the name of NTESS nor the names of its contributors
19 // may be used to endorse or promote products derived from this
20 // software without specific prior written permission.
21 //
22 // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
23 // "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
24 // LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
25 // A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
26 // OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
27 // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
28 // LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
29 // DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
30 // THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
31 // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
32 // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
33
34 #include "stk_util/stk_config.h" // for STK_HAS_MPI
35 #ifdef STK_HAS_MPI
36
37 #include "gtest/gtest.h"
38 #include "stk_util/parallel/MPICommKey.hpp"
39
40 namespace {
41
42 class KeyManagerTester : public ::testing::Test
43 {
44 protected:
45 KeyManagerTester()
46 {
47 comm1 = MPI_COMM_WORLD;
48 MPI_Comm_dup(comm1, &comm2);
49 }
50
51 ~KeyManagerTester()
52 {
53 MPI_Comm_free(&comm2);
54 }
55
56 MPI_Comm comm1;
57 MPI_Comm comm2;
58 stk::impl::MPIKeyManager keyManager;
59 };
60
61 class CallbackChecker
62 {
63 public:
64 int callcount = 0;
65
66 void set_called(MPI_Comm) { callcount++; }
67 };
68
69 } // namespace
70
71 #ifndef MPI_KEY_MANAGER_COMM_DESTRUCTOR_CALLBACK_BROKEN
72 namespace {
73 MPI_Request req;
74 bool is_complete = false;
75 int comm_destructor_callback(MPI_Comm, int, void*, void*)
76 {
77 MPI_Wait(&req, MPI_STATUS_IGNORE);
78 is_complete = true;
79
80 return MPI_SUCCESS;
81 }
82 }
83
84 TEST(KeyManager, IntelMPIBug)
85 {
86 MPI_Comm comm2;
87 MPI_Comm_dup(MPI_COMM_WORLD, &comm2);
88 int keyval;
89 MPI_Comm_create_keyval(MPI_COMM_NULL_COPY_FN, &comm_destructor_callback, &keyval, nullptr);
90 MPI_Comm_set_attr(comm2, keyval, nullptr);
91
92 is_complete = false;
93 MPI_Ibarrier(comm2, &req);
94 MPI_Comm_free(&comm2);
95 EXPECT_TRUE(is_complete);
96 }
97 #endif
98
99
100 TEST_F(KeyManagerTester, KeyValuesUniqueness)
101 {
102 MPI_Comm comm3, comm4;
103 MPI_Comm_dup(comm1, &comm3);
104 MPI_Comm_dup(comm1, &comm4);
105
106
107 std::vector<stk::impl::MPIKeyManager::CommKey> keyVals;
108 keyVals.push_back(keyManager.get_key(comm1));
109 keyVals.push_back(keyManager.get_key(comm2));
110 keyVals.push_back(keyManager.get_key(comm3));
111 keyVals.push_back(keyManager.get_key(comm4));
112
113 for (unsigned int i=0; i < keyVals.size(); ++i)
114 for (unsigned int j=0; j < keyVals.size(); ++j)
115 if (i == j)
116 EXPECT_EQ(keyVals[i], keyVals[j]);
117 else
118 EXPECT_NE(keyVals[i], keyVals[j]);
119
120 MPI_Comm_free(&comm3);
121 MPI_Comm_free(&comm4);
122 }
123
124
125 TEST_F(KeyManagerTester, getTagConst)
126 {
127 auto key1 = keyManager.get_key(comm1);
128 auto key2 = keyManager.get_key(comm2);
129
130 const stk::impl::MPIKeyManager& keyManagerConst = keyManager;
131
132 EXPECT_EQ(key1, keyManagerConst.get_key(comm1));
133 EXPECT_EQ(key2, keyManagerConst.get_key(comm2));
134
135 MPI_Comm comm3;
136 MPI_Comm_dup(comm1, &comm3);
137 EXPECT_ANY_THROW(keyManagerConst.get_key(comm3));
138 MPI_Comm_free(&comm3);
139 }
140
141
142 TEST_F(KeyManagerTester, KeyReuse)
143 {
144 MPI_Comm comm3;
145 MPI_Comm_dup(comm1, &comm3);
146
147 auto key1 = keyManager.get_key(comm1);
148 auto key2 = keyManager.get_key(comm2);
149 keyManager.get_key(comm3);
150
151 MPI_Comm_free(&comm3);
152 MPI_Comm_dup(comm1, &comm3);
153
154 auto key4 = keyManager.get_key(comm3);
155
156 EXPECT_NE(key1, key4);
157 EXPECT_NE(key2, key4);
158
159 MPI_Comm_free(&comm3);
160 }
161
162
163 TEST_F(KeyManagerTester, SplitCommSubsetOfProcs)
164 {
165 MPI_Comm comm3;
166 int myrank;
167 MPI_Comm_rank(comm1, &myrank);
168 int color = myrank == 0 ? MPI_UNDEFINED : 1;
169 MPI_Comm_split(comm1, color, 0, &comm3);
170
171 auto key1 = keyManager.get_key(comm1);
172 stk::impl::MPIKeyManager::CommKey key3;
173 if (color != MPI_UNDEFINED)
174 key3 = keyManager.get_key(comm3);
175 auto key2 = keyManager.get_key(comm2);
176
177 EXPECT_NE(key1, key2);
178 if (color != MPI_UNDEFINED)
179 {
180 EXPECT_NE(key1, key3);
181 EXPECT_NE(key2, key3);
182
183 MPI_Comm_free(&comm3);
184 }
185 }
186
187
188 TEST_F(KeyManagerTester, Callback)
189 {
190 MPI_Comm comm3;
191 MPI_Comm_dup(comm1, &comm3);
192
193 CallbackChecker checker3;
194 keyManager.get_key(comm3);
195
196 EXPECT_EQ(checker3.callcount, 0);
197 auto callerUID = keyManager.get_UID();
198 keyManager.register_callback(comm3, callerUID,
199 std::bind(&CallbackChecker::set_called, &checker3, std::placeholders::_1));
200 MPI_Comm_free(&comm3);
201 EXPECT_EQ(checker3.callcount, 1);
202
203 }
204
205
206 TEST(KeyManager, CommNeverFreed)
207 {
208 MPI_Comm comm = MPI_COMM_WORLD;
209 CallbackChecker checker;
210
211 {
212 stk::impl::MPIKeyManager keyManager;
213 keyManager.get_key(comm);
214 auto callerUID = keyManager.get_UID();
215 keyManager.register_callback(comm, callerUID,
216 std::bind(&CallbackChecker::set_called, &checker, std::placeholders::_1));
217
218 EXPECT_EQ(checker.callcount, 0);
219 }
220
221 EXPECT_EQ(checker.callcount, 1);
222 } // comm is never freed, but MPIKeyManager shouldn't segfault
223
224
225 TEST(KeyManager, CommFreedAfterManager)
226 {
227 MPI_Comm comm3;
228 MPI_Comm_dup(MPI_COMM_WORLD, &comm3);
229
230 {
231 stk::impl::MPIKeyManager keyManager;
232 keyManager.get_key(comm3);
233 } // keyManager destructor runs here, shouldn't segfault
234
235 MPI_Comm_free(&comm3);
236 }
237
238
239 TEST(KeyManager, CommFreedAfterManagerWithCallback)
240 {
241 MPI_Comm comm3;
242 MPI_Comm_dup(MPI_COMM_WORLD, &comm3);
243 CallbackChecker checker3;
244
245 {
246 stk::impl::MPIKeyManager keyManager;
247 keyManager.get_key(comm3);
248 auto callerUID = keyManager.get_UID();
249 keyManager.register_callback(comm3, callerUID,
250 std::bind(&CallbackChecker::set_called, &checker3, std::placeholders::_1));
251 }
252
253 EXPECT_EQ(checker3.callcount, 1);
254 MPI_Comm_free(&comm3);
255 EXPECT_EQ(checker3.callcount, 1);
256 }
257
258
259 TEST(KeyManager, CommFreedCallback)
260 {
261 MPI_Comm comm3;
262 MPI_Comm_dup(MPI_COMM_WORLD, &comm3);
263 CallbackChecker checker3;
264
265 {
266 stk::impl::MPIKeyManager keyManager;
267 keyManager.get_key(comm3);
268 auto callerUID = keyManager.get_UID();
269 keyManager.register_callback(comm3, callerUID,
270 std::bind(&CallbackChecker::set_called, &checker3, std::placeholders::_1));
271
272 EXPECT_EQ(checker3.callcount, 0);
273 MPI_Comm_free(&comm3);
274 EXPECT_EQ(checker3.callcount, 1);
275 }
276
277
278 EXPECT_EQ(checker3.callcount, 1);
279 }
280
281
282 TEST(KeyManager, CallbackImmediately)
283 {
284 MPI_Comm comm3;
285 MPI_Comm_dup(MPI_COMM_WORLD, &comm3);
286 CallbackChecker checker3;
287
288 {
289 stk::impl::MPIKeyManager keyManager;
290 keyManager.get_key(comm3);
291 auto callerUID = keyManager.get_UID();
292 keyManager.register_callback(comm3,
293 callerUID, std::bind(&CallbackChecker::set_called, &checker3, std::placeholders::_1));
294 keyManager.execute_callbacks_immediately(callerUID);
295
296 EXPECT_EQ(checker3.callcount, 1);
297 }
298
299 EXPECT_EQ(checker3.callcount, 1);
300 MPI_Comm_free(&comm3);
301 EXPECT_EQ(checker3.callcount, 1);
302 }
303
304
305 TEST(KeyManager, UnregisterCallbacks)
306 {
307 MPI_Comm comm3;
308 MPI_Comm_dup(MPI_COMM_WORLD, &comm3);
309 CallbackChecker checker3;
310
311 {
312 stk::impl::MPIKeyManager keyManager;
313 keyManager.get_key(comm3);
314 auto callerUID = keyManager.get_UID();
315 keyManager.register_callback(comm3,
316 callerUID, std::bind(&CallbackChecker::set_called, &checker3, std::placeholders::_1));
317 keyManager.unregister_callbacks(callerUID);
318
319 EXPECT_EQ(checker3.callcount, 0);
320 }
321
322 EXPECT_EQ(checker3.callcount, 0);
323 MPI_Comm_free(&comm3);
324 EXPECT_EQ(checker3.callcount, 0);
325 }
326
327
328 TEST(KeyManager, MultipleCallbacks)
329 {
330 MPI_Comm comm3;
331 MPI_Comm_dup(MPI_COMM_WORLD, &comm3);
332 CallbackChecker checker1, checker2, checker3;
333
334 auto callback1 = std::bind(&CallbackChecker::set_called, &checker1, std::placeholders::_1);
335 auto callback2 = std::bind(&CallbackChecker::set_called, &checker2, std::placeholders::_1);
336 auto callback3 = std::bind(&CallbackChecker::set_called, &checker3, std::placeholders::_1);
337 {
338 stk::impl::MPIKeyManager keyManager;
339 keyManager.get_key(comm3);
340 auto callerUID1 = keyManager.get_UID();
341 auto callerUID2 = keyManager.get_UID();
342 auto callerUID3 = keyManager.get_UID();
343 keyManager.register_callback(comm3, callerUID1, callback1);
344 keyManager.register_callback(comm3, callerUID2, callback2);
345 keyManager.register_callback(comm3, callerUID3, callback3);
346
347 keyManager.execute_callbacks_immediately(callerUID1);
348 EXPECT_EQ(checker1.callcount, 1);
349 EXPECT_EQ(checker2.callcount, 0);
350 EXPECT_EQ(checker3.callcount, 0);
351
352 keyManager.unregister_callbacks(callerUID2);
353 EXPECT_EQ(checker1.callcount, 1);
354 EXPECT_EQ(checker2.callcount, 0);
355 EXPECT_EQ(checker3.callcount, 0);
356 }
357
358 EXPECT_EQ(checker1.callcount, 1);
359 EXPECT_EQ(checker2.callcount, 0);
360 EXPECT_EQ(checker3.callcount, 1);
361
362 MPI_Comm_free(&comm3);
363 EXPECT_EQ(checker1.callcount, 1);
364 EXPECT_EQ(checker2.callcount, 0);
365 EXPECT_EQ(checker3.callcount, 1);
366 }
367
368
369 TEST(KeyManager, OrderedKeys)
370 {
371 MPI_Comm comm1=MPI_COMM_WORLD, comm2, comm3, comm4;
372 int myRank, commSize;
373 MPI_Comm_rank(comm1, &myRank);
374 MPI_Comm_size(comm1, &commSize);
375
376 int color = myRank == commSize-1 ? MPI_UNDEFINED : 1;
377 MPI_Comm_split(comm1, color, 0, &comm2);
378 MPI_Comm_dup(comm1, &comm3);
379 MPI_Comm_dup(comm1, &comm4);
380
381 auto keyManager = std::make_shared<stk::impl::MPIKeyManager>();
382 keyManager->get_key(comm1);
383 if (color != MPI_UNDEFINED)
384 keyManager->get_key(comm2);
385
386 keyManager->get_key(comm3);
387 if (color != MPI_UNDEFINED)
388 MPI_Comm_free(&comm2);
389
390 keyManager->get_key(comm4);
391
392 std::vector<MPI_Comm> comms{comm1, comm3, comm4};
393 std::sort(comms.begin(), comms.end(), stk::impl::CommCompare(keyManager));
394
395 // if this doesn't hang, the comms are in the same order on all procs
396 for (auto& comm : comms)
397 MPI_Barrier(comm);
398
399 MPI_Comm_free(&comm3);
400 MPI_Comm_free(&comm4);
401 }
402
403 #endif