Debian Codegraph @

packages/stk/stk_unit_tests/stk_util/parallel/UnitTestCommNeighbors.cpp

raw
1 // Copyright 2002 - 2008, 2010, 2011 National Technology Engineering
2 // Solutions of Sandia, LLC (NTESS). Under the terms of Contract
3 // DE-NA0003525 with NTESS, the U.S. Government retains certain rights
4 // in this software.
5 //
6 // Redistribution and use in source and binary forms, with or without
7 // modification, are permitted provided that the following conditions are
8 // met:
9 //
10 // * Redistributions of source code must retain the above copyright
11 // notice, this list of conditions and the following disclaimer.
12 //
13 // * Redistributions in binary form must reproduce the above
14 // copyright notice, this list of conditions and the following
15 // disclaimer in the documentation and/or other materials provided
16 // with the distribution.
17 //
18 // * Neither the name of NTESS nor the names of its contributors
19 // may be used to endorse or promote products derived from this
20 // software without specific prior written permission.
21 //
22 // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
23 // "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
24 // LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
25 // A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
26 // OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
27 // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
28 // LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
29 // DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
30 // THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
31 // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
32 // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
33 //
34
35 #include "gtest/gtest.h"
36 #include "stk_util/parallel/CommBufferV.hpp" // for CommBufferV
37 #include "stk_util/parallel/CommNeighbors.hpp" // for CommNeighbors, STK_MPI_SUPPORTS_NEIGHBOR_...
38 #include "stk_util/parallel/Parallel.hpp" // for parallel_machine_rank, parallel_machine_size
39 #include "stk_util/stk_config.h" // for STK_HAS_MPI
40 #include <cstddef> // for size_t
41 #include <vector> // for vector
42
43 #if defined ( STK_HAS_MPI )
44
45 std::vector<int> get_neighbor_procs(int numAllProcs, int localProc, int numNeighbors)
46 {
47 std::vector<int> neighbors;
48 for(int proc=localProc-numNeighbors/2; proc<=localProc+numNeighbors/2; ++proc) {
49 if (proc >= 0 && proc != localProc && proc < numAllProcs) {
50 neighbors.push_back(proc);
51 }
52 }
53 return neighbors;
54 }
55
56 TEST(Parallel, CommNeighborsOneSided_Raw_MPI)
57 {
58 #ifdef OMPI_MAJOR_VERSION
59 #if OMPI_MAJOR_VERSION < 2
60 #undef STK_MPI_SUPPORTS_NEIGHBOR_COMM
61 #endif
62 #endif
63
64 #if !defined(STK_MPI_SUPPORTS_NEIGHBOR_COMM)
65 return;
66 #else
67
68 MPI_Comm comm = MPI_COMM_WORLD;
69 int numProcs = 1;
70 MPI_Comm_size(comm, &numProcs);
71 if (numProcs != 2) {
72 return;
73 }
74
75 //this test will send from proc 0 to proc 1.
76 //proc 0 will not recv anything and proc 1 will not send anything.
77 //'sendProcs' means 'procs that I will send to'
78 //'recvprocs' means 'procs that I will recv from'
79
80 int localProc = 0;
81 MPI_Comm_rank(comm, &localProc);
82 int otherProc = 1 - localProc;
83 std::vector<int> neighbors = {otherProc};
84 std::vector<int> sendProcs = {otherProc};
85 std::vector<int> recvProcs = {otherProc};
86 if (localProc==0) {
87 recvProcs.clear();
88 }
89 else {
90 sendProcs.clear();
91 }
92
93 MPI_Info info;
94 MPI_Info_create(&info);
95 int reorder = 0;
96 const int* weights = (int*)MPI_UNWEIGHTED;
97
98 MPI_Comm neighborComm;
99 MPI_Dist_graph_create_adjacent(comm,
100 neighbors.size(), neighbors.data(), weights,
101 neighbors.size(), neighbors.data(), weights,
102 info, reorder, &neighborComm);
103 MPI_Info_free(&info);
104
105 int numItems = 5;
106 std::vector<double> sendBuf(numItems, 0), recvBuf;
107 std::vector<int> sendCounts(neighbors.size(), 0), recvCounts(neighbors.size());
108 std::vector<int> sendDispls(neighbors.size(), 0), recvDispls(neighbors.size());
109
110 for(size_t i=0; i<sendProcs.size(); ++i) {
111 sendCounts[i] = numItems;
112 }
113
114 if (localProc == 0) {
115 for(int i=0; i<numItems; ++i) {
116 sendBuf[i] = (localProc + 1 + i);
117 }
118 }
119
120 MPI_Neighbor_alltoall((void*)sendCounts.data(), 1, MPI_INT,
121 (void*)recvCounts.data(), 1, MPI_INT, neighborComm);
122
123 int totalRecv = 0;
124 recvDispls.resize(recvCounts.size());
125 for(size_t i=0; i<recvCounts.size(); ++i) {
126 recvDispls[i] = totalRecv;
127 totalRecv += recvCounts[i];
128 }
129 recvBuf.resize(totalRecv);
130
131 MPI_Neighbor_alltoallv(
132 (void*)sendBuf.data(), sendCounts.data(), sendDispls.data(), MPI_DOUBLE,
133 (void*)recvBuf.data(), recvCounts.data(), recvDispls.data(), MPI_DOUBLE, neighborComm);
134
135 for(unsigned i=0; i<recvProcs.size(); ++i) {
136 EXPECT_EQ(1, localProc);//should only recv on proc 1
137
138 const double* recvdFromProcI = recvBuf.data() + recvDispls[i];
139 EXPECT_EQ(numItems, recvCounts[i]);
140
141 for(int item=0; item<numItems; ++item) {
142 double val = recvdFromProcI[item];
143 double expected = recvProcs[i] + 1 + item;
144 EXPECT_EQ(expected, val);
145 }
146 }
147 #endif
148 }
149
150 TEST(Parallel, CommNeighborsOneSided)
151 {
152 stk::ParallelMachine comm = MPI_COMM_WORLD;
153 int numProcs = stk::parallel_machine_size(comm);
154 if (numProcs != 2) {
155 return;
156 }
157
158 int localProc = stk::parallel_machine_rank(comm);
159 int otherProc = 1 - localProc;
160 std::vector<int> sendProcs = {otherProc};
161 std::vector<int> recvProcs = {otherProc};
162 if (localProc==0) {
163 recvProcs.clear();
164 }
165 else {
166 sendProcs.clear();
167 }
168
169 stk::CommNeighbors commNeighbors(comm, sendProcs, recvProcs);
170
171 int numItems = 5;
172 for(int proc : sendProcs) {
173 stk::CommBufferV& buf = commNeighbors.send_buffer(proc);
174 for(int i=0; i<numItems; ++i) {
175 buf.pack<double>(localProc+1+i);
176 }
177 }
178
179 commNeighbors.communicate();
180
181 for(int proc : recvProcs) {
182 stk::CommBufferV& buf = commNeighbors.recv_buffer(proc);
183
184 int remaining = buf.size_in_bytes()/sizeof(double);
185 EXPECT_EQ(numItems, remaining);
186
187 for(int i=0; i<numItems; ++i) {
188 double val = 0;
189 buf.unpack<double>(val);
190 double expected = proc+1+i;
191 EXPECT_EQ(expected, val);
192 }
193 }
194 }
195
196 TEST(Parallel, CommNeighbors)
197 {
198 stk::ParallelMachine comm = MPI_COMM_WORLD;
199 int numProcs = stk::parallel_machine_size(comm);
200 if (numProcs == 1) {
201 return;
202 }
203
204 int myProc = stk::parallel_machine_rank(comm);
205 int numNeighbors = 5;
206 std::vector<int> neighborProcs = get_neighbor_procs(numProcs, myProc, numNeighbors);
207 int numItems = 20;
208
209 stk::parallel_machine_barrier(comm);
210
211 stk::CommNeighbors commNeighbors(comm, neighborProcs);
212
213 EXPECT_EQ(numProcs, commNeighbors.parallel_size());
214 EXPECT_EQ(myProc, commNeighbors.parallel_rank());
215
216 for(int proc : neighborProcs) {
217 stk::CommBufferV& proc_buff = commNeighbors.send_buffer(proc);
218 for(int i=0; i<numItems; ++i) {
219 proc_buff.pack<double>(myProc+1+i);
220 }
221 }
222
223 commNeighbors.communicate();
224
225 for(int proc : neighborProcs) {
226 stk::CommBufferV& proc_buff = commNeighbors.recv_buffer(proc);
227
228 int remaining = proc_buff.size_in_bytes()/sizeof(double);
229 EXPECT_EQ(numItems, remaining);
230
231 for(int i=0; i<numItems; ++i) {
232 double val = 0;
233 proc_buff.unpack<double>(val);
234 double expected = proc+1+i;
235 EXPECT_EQ(expected, val);
236 }
237 }
238 }
239
240 TEST(Parallel, CommNeighborsResetBuffers) {
241 stk::ParallelMachine comm = MPI_COMM_WORLD;
242 std::vector<int> send, recv;
243 int numProcs = stk::parallel_machine_size(comm);
244 int rank= stk::parallel_machine_rank(comm);
245 if (numProcs == 1) {
246 return;
247 }
248 if(rank == 0) {
249 for(int p = 1; p < numProcs; ++p) {
250 send.push_back(p);
251 }
252 } else {
253 recv.push_back(0);
254 }
255
256 stk::CommNeighbors commNeighbors(comm, send, recv);
257
258 for(int i = 0; i < 3; ++i) {
259 commNeighbors.reset_buffers();
260 for(int p : commNeighbors.send_procs()) {
261 commNeighbors.send_buffer(p).pack<int>(i);
262 }
263
264 commNeighbors.communicate();
265
266 for(int p : commNeighbors.recv_procs()) {
267 int val = 0;
268 commNeighbors.recv_buffer(p).unpack<int>(val);
269 EXPECT_EQ(i, val);
270 EXPECT_EQ(0u, commNeighbors.recv_buffer(p).size_in_bytes());
271 }
272 }
273 }
274
275 #endif