Debian Codegraph @

packages/stk/stk_unit_tests/stk_util/parallel/UnitTestDeviceAwareMPI.cpp

raw
1 // Copyright 2002 - 2008, 2010, 2011 National Technology Engineering
2 // Solutions of Sandia, LLC (NTESS). Under the terms of Contract
3 // DE-NA0003525 with NTESS, the U.S. Government retains certain rights
4 // in this software.
5 //
6 // Redistribution and use in source and binary forms, with or without
7 // modification, are permitted provided that the following conditions are
8 // met:
9 //
10 // * Redistributions of source code must retain the above copyright
11 // notice, this list of conditions and the following disclaimer.
12 //
13 // * Redistributions in binary form must reproduce the above
14 // copyright notice, this list of conditions and the following
15 // disclaimer in the documentation and/or other materials provided
16 // with the distribution.
17 //
18 // * Neither the name of NTESS nor the names of its contributors
19 // may be used to endorse or promote products derived from this
20 // software without specific prior written permission.
21 //
22 // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
23 // "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
24 // LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
25 // A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
26 // OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
27 // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
28 // LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
29 // DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
30 // THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
31 // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
32 // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
33 //
34
35 #include "gtest/gtest.h"
36 #include "stk_util/parallel/Parallel.hpp"
37 #include "stk_util/parallel/DeviceAwareMPI.hpp"
38 #include "stk_util/ngp/NgpSpaces.hpp"
39
40 #ifdef STK_HAS_MPI
41
42 TEST(DeviceAwareMPI, trueIfOpenMPIAndCuda)
43 {
44 #if defined(OMPI_MAJOR_VERSION) && defined(KOKKOS_ENABLE_CUDA)
45 EXPECT_TRUE(stk::have_device_aware_mpi());
46 #else
47 GTEST_SKIP()<<"trueIfOpenMPIAndCuda";
48 #endif
49 }
50
51 TEST(DeviceAwareMPI, trueIfATS2MPIAndCuda)
52 {
53 #if defined(IBM_SPECTRUM_MPI) && defined(KOKKOS_ENABLE_CUDA)
54 EXPECT_TRUE(stk::have_device_aware_mpi());
55 #else
56 GTEST_SKIP()<<"trueIfATS2MPIAndCuda";
57 #endif
58 }
59
60 TEST(DeviceAwareMPI, falseIfOpenMpiButNoCuda)
61 {
62 #if defined(OMPI_MAJOR_VERSION) && !defined(KOKKOS_ENABLE_CUDA)
63 EXPECT_FALSE(stk::have_device_aware_mpi());
64 #else
65 GTEST_SKIP()<<"falseIfOpenMpiButNoCuda";
66 #endif
67 }
68
69 TEST(DeviceAwareMPI, falseIfIntelMpi)
70 {
71 #if defined(I_MPI_VERSION)
72 EXPECT_FALSE(stk::have_device_aware_mpi());
73 #else
74 GTEST_SKIP()<<"falseIfIntel, I_MPI_VERSION not defined";
75 #endif
76 }
77
78 void check_device_aware_mpi_send_recv(MPI_Comm comm)
79 {
80 const int numProcs = stk::parallel_machine_size(comm);
81 ASSERT_EQ(2, numProcs);
82 const int myProc = stk::parallel_machine_rank(comm);
83 const int otherProc = 1 - myProc;
84 const int msgTag = 10101;
85
86 using BufferViewType = Kokkos::View<double*,stk::ngp::MemSpace>;
87 constexpr size_t N = 8;
88 constexpr double tol = 1.e-7;
89 constexpr double goldValue = 3.14159;
90
91 if (myProc == 0) {
92 BufferViewType sendBuf("sendBuf",N);
93 Kokkos::deep_copy(sendBuf, goldValue);
94 EXPECT_EQ(MPI_SUCCESS, MPI_Send(sendBuf.data(), N, MPI_DOUBLE, otherProc, msgTag, comm));
95 }
96 else {
97 BufferViewType recvBuf("recvBuf",N);
98 Kokkos::deep_copy(recvBuf, 0.0);
99 MPI_Status status;
100 EXPECT_EQ( MPI_SUCCESS, MPI_Recv(recvBuf.data(), N, MPI_DOUBLE, otherProc, msgTag, comm, &status));
101
102 BufferViewType::HostMirror hostRecvBuf = Kokkos::create_mirror_view(recvBuf);
103 Kokkos::deep_copy(hostRecvBuf, recvBuf);
104 for(size_t i=0; i<N; ++i) {
105 EXPECT_NEAR(goldValue, hostRecvBuf(i), tol);
106 }
107 }
108 }
109
110 TEST(DeviceAwareMPI, basicSendRecv_np2)
111 {
112 if (stk::have_device_aware_mpi() && stk::parallel_machine_size(MPI_COMM_WORLD) == 2) {
113 check_device_aware_mpi_send_recv(MPI_COMM_WORLD);
114 }
115 else {
116 GTEST_SKIP();
117 }
118 }
119
120 void check_device_aware_mpi_isend_irecv_waitany(MPI_Comm comm)
121 {
122 const int numProcs = stk::parallel_machine_size(comm);
123 ASSERT_EQ(2, numProcs);
124 const int myProc = stk::parallel_machine_rank(comm);
125 const int otherProc = 1 - myProc;
126 const int msgTag = 10101;
127
128 using BufferViewType = Kokkos::View<double*,stk::ngp::MemSpace>;
129 constexpr size_t N = 4200;
130 constexpr double tol = 1.e-7;
131 constexpr double goldValue = 3.14159;
132
133 const int numCommProcs = 1;
134 std::vector<MPI_Request> sendRequests(numCommProcs);
135 std::vector<MPI_Request> recvRequests(numCommProcs);
136 std::vector<MPI_Status> statuses(numCommProcs);
137
138 BufferViewType sendBuf("sendBuf",N);
139 BufferViewType recvBuf("recvBuf",N);
140
141 stk::parallel_machine_barrier(comm);
142
143 if (myProc == 1) {
144 Kokkos::deep_copy(recvBuf, 0.0); //theoretically unnecessary since Views initialize by default.
145 EXPECT_EQ(MPI_SUCCESS, MPI_Irecv(recvBuf.data(), N, MPI_DOUBLE, otherProc, msgTag, comm, &recvRequests[0]));
146 }
147 if (myProc == 0) {
148 Kokkos::deep_copy(sendBuf, goldValue);
149 EXPECT_EQ(MPI_SUCCESS, MPI_Isend(sendBuf.data(), N, MPI_DOUBLE, otherProc, msgTag, comm, &sendRequests[0]));
150 }
151
152 Kokkos::fence();
153
154 if (myProc == 1) {
155 int idx = 99;
156 MPI_Waitany(numCommProcs, recvRequests.data(), &idx, MPI_STATUS_IGNORE);
157 EXPECT_EQ(0, idx);
158
159 BufferViewType::HostMirror hostRecvBuf = Kokkos::create_mirror_view(recvBuf);
160 Kokkos::deep_copy(hostRecvBuf, recvBuf);
161 for(size_t i=0; i<N; ++i) {
162 EXPECT_NEAR(goldValue, hostRecvBuf(i), tol);
163 }
164 }
165
166 if (myProc == 0) {
167 MPI_Waitall(numCommProcs, sendRequests.data(), statuses.data());
168 }
169 }
170
171 TEST(DeviceAwareMPI, basicIsendIrecv_np2)
172 {
173 if (stk::have_device_aware_mpi() && stk::parallel_machine_size(MPI_COMM_WORLD) == 2) {
174 check_device_aware_mpi_isend_irecv_waitany(MPI_COMM_WORLD);
175 }
176 else {
177 GTEST_SKIP();
178 }
179 }
180
181 #endif
182