Debian Codegraph @

packages/stk/stk_unit_tests/stk_util/parallel/UnitTestImprobeAll.cpp

raw
1 #include "mpi.h"
2 #include <vector>
3 #include <cassert>
4 #include <stdexcept>
5 #include <iostream>
6
7 // comment this out to build this file as its own executable
8 #define BUILDING_IN_STK
9
10 #ifdef BUILDING_IN_STK
11 #include "gtest/gtest.h"
12 #endif
13
14 namespace {
15
16 using DType = int;
17 const int NVals = 4096; // use fixed number of elements to simplify this
18 // example. In the real code the number of
19 // elements is unknown, which is why we use
20 // MPI_Improbe
21
22 // compute a unique-ish value (given the receiving rank and the index
23 // in the buffer, we can always compute who the sender is)
24 DType getValue(DType rank_src, DType rank_dest, DType idx)
25 {
26 return rank_src + 2*rank_dest + 3*idx;
27 }
28
29 std::vector<std::vector<DType>> getSendData(MPI_Comm comm)
30 {
31 int comm_size, comm_rank;
32 MPI_Comm_size(comm, &comm_size);
33 MPI_Comm_rank(comm, &comm_rank);
34
35 std::vector<std::vector<DType>> send_bufs(comm_size);
36 for (int dest=0; dest < comm_size; ++dest)
37 {
38 send_bufs[dest].resize(NVals);
39 for (int j=0; j < NVals; ++j)
40 send_bufs[dest][j] = getValue(comm_rank, dest, j);
41 }
42
43 return send_bufs;
44 }
45
46 void checkRecvData(MPI_Comm comm, std::vector<std::vector<DType>>& recv_bufs)
47 {
48 int comm_size, comm_rank;
49 MPI_Comm_size(comm, &comm_size);
50 MPI_Comm_rank(comm, &comm_rank);
51
52 for (int src=0; src < comm_size; ++src)
53 for (int j=0; j < NVals; ++j)
54 if (recv_bufs[src][j] != getValue(src, comm_rank, j))
55 throw std::runtime_error("error in received data");
56 }
57
58 void doCommunication(MPI_Comm comm)
59 {
60 MPI_Barrier(comm);
61 int comm_size, comm_rank, tag=7;
62 MPI_Comm_size(comm, &comm_size);
63 MPI_Comm_rank(comm, &comm_rank);
64
65
66 std::vector<std::vector<DType>> send_bufs = getSendData(comm);
67 std::vector<std::vector<DType>> recv_bufs(comm_size);
68 std::vector<MPI_Request> send_reqs(comm_size), recv_reqs(comm_size);
69
70 // launch sends
71 for (int dest=0; dest < comm_size; ++dest)
72 MPI_Isend(send_bufs[dest].data(), NVals*sizeof(DType), MPI_BYTE, dest, tag, comm, &(send_reqs[dest]));
73
74
75 // using MPI_Irecv works
76 /*
77 for (int src=0; src < comm_size; ++src)
78 {
79 recv_bufs[src].resize(NVals);
80 MPI_Irecv(recv_bufs[src].data(), NVals*sizeof(DType), MPI_BYTE, src, tag, comm, &(recv_reqs[src]));
81 }
82 */
83
84 // using MPI_Improbe does not work
85
86 // Probe for receives
87 int nreceived = 0;
88 while (true)
89 {
90 int flag=false;
91 MPI_Message message;
92 MPI_Status status;
93 MPI_Improbe(MPI_ANY_SOURCE, tag, comm, &flag, &message, &status);
94
95 if (flag)
96 {
97 int src=status.MPI_SOURCE, count=0;
98 MPI_Get_count(&status, MPI_BYTE, &count);
99 assert(count % sizeof(DType) == 0);
100 assert(count == NVals * sizeof(DType));
101 //std::cout << "receiving " << count << " bytes from rank " << src << std::endl;
102
103 recv_bufs[src].resize(count / sizeof(DType));
104 MPI_Imrecv(recv_bufs[src].data(), count, MPI_BYTE, &message, &(recv_reqs[src]));
105 nreceived++;
106 }
107
108 if (nreceived == comm_size)
109 break;
110 }
111
112 MPI_Waitall(send_reqs.size(), send_reqs.data(), MPI_STATUSES_IGNORE);
113 MPI_Waitall(recv_reqs.size(), recv_reqs.data(), MPI_STATUSES_IGNORE);
114 MPI_Barrier(comm);
115 checkRecvData(comm, recv_bufs);
116 }
117
118 } // namespace
119
120
121 #ifdef BUILDING_IN_STK
122 TEST(Improbe, Improbe_All)
123 {
124 doCommunication(MPI_COMM_WORLD);
125 }
126
127 #else
128
129 int main(int argc, char* argv[])
130 {
131 MPI_Init(&argc, &argv);
132
133 doCommunication(MPI_COMM_WORLD);
134
135 MPI_Finalize();
136 }
137 #endif