Debian Codegraph @

packages/stk/stk_integration_tests/stk_util/IntegrationTestCommSparse.cpp

raw
1 #include "gtest/gtest.h"
2 #include "stk_util/parallel/CommSparse.hpp"
3 #include "stk_util/parallel/DataExchangeKnownPatternUserDataNonBlocking.hpp"
4 #include "stk_util/parallel/DataExchangeUnknownPatternNonBlocking.hpp"
5 #include "stk_util/parallel/DataExchangeKnownPatternNonBlocking.hpp"
6
7 namespace {
8 using DataType = long long;
9 // make message large enough to not fit in any int (signed or unsigned)
10 constexpr size_t approxMsgSizeInBytes = (size_t(1) << 32) + 2*sizeof(DataType);
11
12 constexpr size_t numElements = approxMsgSizeInBytes / sizeof(DataType);
13 constexpr size_t msgSizeInBytes = numElements * sizeof(DataType);
14 }
15
16 TEST(CommSparse, LargerThan2GB)
17 {
18 if (stk::parallel_machine_size(stk::parallel_machine_world()) != 2)
19 {
20 GTEST_SKIP();
21 }
22
23 stk::CommSparse comm(stk::parallel_machine_world());
24 int myrank = stk::parallel_machine_rank(stk::parallel_machine_world());
25 constexpr int senderRank = 0;
26 constexpr int receiverRank = 1;
27
28 for (int phase=0; phase < 2; ++phase)
29 {
30 if (myrank == senderRank)
31 {
32 for (size_t i=0; i < numElements; ++i)
33 {
34 comm.send_buffer(receiverRank).pack(DataType(i));
35 }
36 }
37
38 if (phase == 0)
39 {
40 comm.allocate_buffers();
41 if (myrank == senderRank)
42 {
43 EXPECT_EQ(comm.send_buffer(receiverRank).capacity(), msgSizeInBytes);
44 }
45 } else
46 {
47 comm.communicate();
48 }
49 }
50
51 if (myrank == 1)
52 {
53 EXPECT_EQ(comm.recv_buffer(senderRank).capacity(), msgSizeInBytes);
54 for (size_t i=0; i < numElements; ++i)
55 {
56 DataType val;
57 comm.recv_buffer(senderRank).unpack(val);
58 EXPECT_EQ(val, DataType(i));
59 }
60 }
61 }
62
63 TEST(DataExchangeUnknownPatternNonBlocking, LargerThan2GB)
64 {
65 if (stk::parallel_machine_size(stk::parallel_machine_world()) != 2)
66 {
67 GTEST_SKIP();
68 }
69
70 stk::DataExchangeUnknownPatternNonBlocking comm(stk::parallel_machine_world());
71 int myrank = stk::parallel_machine_rank(stk::parallel_machine_world());
72 constexpr int senderRank = 0;
73 constexpr int receiverRank = 1;
74
75 for (int repeat=0; repeat < 3; ++repeat)
76 {
77 std::vector<std::vector<DataType>> sendBufs(2);
78 std::vector<std::vector<DataType>> recvBufs(2);
79 if (myrank == senderRank)
80 {
81 sendBufs[receiverRank].resize(numElements);
82 for (size_t i=0; i < numElements; ++i)
83 {
84 sendBufs[receiverRank][i] = i;
85 }
86 }
87
88 comm.start_nonblocking(sendBufs, recvBufs);
89 comm.post_nonblocking_receives(recvBufs);
90
91 auto unpacker = [&](int rank, const std::vector<DataType>& recv_buf)
92 {
93 EXPECT_EQ(rank, senderRank);
94 EXPECT_EQ(recv_buf.size(), numElements);
95 for (size_t i=0; i < numElements; ++i)
96 {
97 EXPECT_EQ(recv_buf[i], DataType(i));
98 }
99 };
100
101 comm.complete_receives(recvBufs, unpacker);
102 comm.complete_sends();
103 }
104 }
105
106 TEST(DataExchangeKnownPatternNonBlocking, LargerThan2GB)
107 {
108 if (stk::parallel_machine_size(stk::parallel_machine_world()) != 2)
109 {
110 GTEST_SKIP();
111 }
112
113 stk::DataExchangeKnownPatternNonBlocking comm(stk::parallel_machine_world());
114 int myrank = stk::parallel_machine_rank(stk::parallel_machine_world());
115 constexpr int senderRank = 0;
116 constexpr int receiverRank = 1;
117
118 for (int repeat = 0; repeat < 3; ++repeat)
119 {
120 std::vector<std::vector<DataType>> sendBufs(2);
121 std::vector<std::vector<DataType>> recvBufs(2);
122 if (myrank == senderRank)
123 {
124 sendBufs[receiverRank].reserve(numElements);
125 for (size_t i=0; i < numElements; ++i)
126 {
127 sendBufs[receiverRank].push_back(i);
128 }
129 } else
130 {
131 recvBufs[senderRank].resize(numElements);
132 }
133
134 comm.start_nonblocking(sendBufs, recvBufs);
135
136 auto unpacker = [&](int rank, const std::vector<DataType>& recv_buf)
137 {
138 EXPECT_EQ(rank, senderRank);
139 EXPECT_EQ(recv_buf.size(), numElements);
140 for (size_t i=0; i < numElements; ++i)
141 {
142 EXPECT_EQ(recv_buf[i], DataType(i));
143 }
144 };
145
146 comm.complete_receives(recvBufs, unpacker);
147 comm.complete_sends();
148 }
149 }
150
151 TEST(DataExchangeKnownPatternUserDataNonBlocking, LargerThan2GB)
152 {
153 if (stk::parallel_machine_size(stk::parallel_machine_world()) != 2)
154 {
155 GTEST_SKIP();
156 }
157
158 stk::DataExchangeKnownPatternUserDataNonBlocking comm(stk::parallel_machine_world());
159 int myrank = stk::parallel_machine_rank(stk::parallel_machine_world());
160 constexpr int senderRank = 0;
161 constexpr int receiverRank = 1;
162
163 for (int repeat = 0; repeat < 3; ++repeat)
164 {
165 std::vector<DataType> allSendBufs(numElements), allRecvBufs(numElements);
166 std::vector<stk::PointerAndSize> sendBufPointers, recvBufPointers;
167 std::vector<int> sendRanks, recvRanks;
168 if (myrank == senderRank)
169 {
170 for (size_t i=0; i < numElements; ++i)
171 {
172 allSendBufs[i] = i;
173 }
174 sendBufPointers.emplace_back(reinterpret_cast<unsigned char*>(allSendBufs.data()),
175 numElements * sizeof(DataType));
176 sendRanks.push_back(receiverRank);
177 } else
178 {
179 recvBufPointers.emplace_back(reinterpret_cast<unsigned char*>(allRecvBufs.data()),
180 numElements * sizeof(DataType));
181 recvRanks.push_back(senderRank);
182 }
183
184 comm.start_nonblocking(sendBufPointers, sendRanks, recvBufPointers, recvRanks);
185
186 auto unpacker = [&](int rank, const stk::PointerAndSize& recv_buf)
187 {
188 EXPECT_EQ(rank, senderRank);
189 EXPECT_EQ(recv_buf.size, numElements * sizeof(DataType));
190 const DataType* ptr = reinterpret_cast<const DataType*>(recv_buf.ptr);
191 for (size_t i=0; i < numElements; ++i)
192 {
193 EXPECT_EQ(ptr[i], DataType(i));
194 EXPECT_EQ(allRecvBufs[i], DataType(i));
195 }
196 };
197
198 comm.complete_receives(recvBufPointers, recvRanks, unpacker);
199 comm.complete_sends();
200 }
201 }
202