Debian Codegraph @

packages/stk/stk_unit_tests/stk_util/parallel/UnitTestIreduceReplacement.cpp

raw
1 #include "gtest/gtest.h"
2 #include "stk_util/parallel/IallreduceReplacement.hpp"
3 #include "stk_util/parallel/Parallel.hpp"
4
5 namespace
6 {
7
8 void sum_op(void* invec, void* inoutvec, int* len, MPI_Datatype* /*datatype*/)
9 {
10 int* invec_int = reinterpret_cast<int*>(invec);
11 int* inoutvec_int = reinterpret_cast<int*>(inoutvec);
12 for (int i=0; i < *len; ++i)
13 inoutvec_int[i] = invec_int[i] + inoutvec_int[i];
14 }
15
16 double test_allreduce_mpi(int numValues)
17 {
18 int num_iters = 100;
19 double t_start = 0;
20
21 std::vector<int> input_data(numValues, 1), output_data(numValues, 42);
22 MPI_Op mpi_op;
23 MPI_Op_create(&sum_op, true, &mpi_op);
24 for (int i=0; i < num_iters; ++i)
25 {
26 if (i == 1)
27 t_start = MPI_Wtime();
28
29 MPI_Request req;
30 //MPI_Ibarrier(MPI_COMM_WORLD, &req);
31 MPI_Iallreduce(input_data.data(), output_data.data(), numValues, MPI_INT, mpi_op, MPI_COMM_WORLD, &req);
32 MPI_Wait(&req, MPI_STATUS_IGNORE);
33 }
34
35 double elapsed_time = (MPI_Wtime() - t_start) / (num_iters - 1);
36 double max_time = 0;
37 MPI_Allreduce(&elapsed_time, &max_time, 1, MPI_DOUBLE, MPI_MAX, MPI_COMM_WORLD);
38
39 return max_time;
40 }
41
42
43 void test_allreduce(stk::impl::IAllreduceReplacement<int>& allreduce, int numValues)
44 {
45 auto op = [](const int& lhs, const int& rhs) { return lhs + rhs; };
46 std::vector<int> input_data(numValues, stk::parallel_machine_rank(MPI_COMM_WORLD)), output_data(numValues, 42),
47 expected_data(numValues);
48
49 int commRank = stk::parallel_machine_rank(MPI_COMM_WORLD);
50 int commSize = stk::parallel_machine_size(MPI_COMM_WORLD);
51
52 if (commRank == 0)
53 std::cout << "\nTesting size " << numValues << std::endl;
54
55
56 for (int i=0; i < numValues; ++i)
57 {
58 input_data[i] = commRank + i;
59 expected_data[i] = i*commSize + commSize * (commSize - 1) / 2;
60 }
61
62 int num_iters = 100;
63 double t_start = 0;
64
65 for (int i=0; i < num_iters; ++i)
66 {
67 if (i == 1)
68 t_start = MPI_Wtime();
69
70 allreduce.startReduction(op, input_data, output_data);
71 allreduce.finishReduction();
72 }
73
74 double elapsed_time = (MPI_Wtime() - t_start) / (num_iters - 1);
75 double max_time = 0;
76 MPI_Allreduce(&elapsed_time, &max_time, 1, MPI_DOUBLE, MPI_MAX, MPI_COMM_WORLD);
77 if (commRank == 0)
78 std::cout << "New algorithm time = " << max_time << " per iteration" << std::endl;
79
80 for (int i=0; i < numValues; ++i)
81 {
82 EXPECT_EQ(output_data[i], expected_data[i]);
83 }
84
85 double max_time2 = test_allreduce_mpi(numValues);
86
87 if (commRank == 0)
88 std::cout << "MPI algorithm time = " << max_time2 << " per iteration" << ", ratio " << max_time / max_time2 << std::endl;
89 }
90 }
91
92
93
94
95 TEST(IallreduceReplacement, ReductionFactor2)
96 {
97 stk::impl::IAllreduceReplacement<int> allreduce(MPI_COMM_WORLD, MPI_INT, 666, 2);
98 MPI_Barrier(MPI_COMM_WORLD);
99 test_allreduce(allreduce, 1);
100 test_allreduce(allreduce, 1);
101 test_allreduce(allreduce, 2);
102 test_allreduce(allreduce, 10);
103 test_allreduce(allreduce, 100);
104 test_allreduce(allreduce, 1000);
105 test_allreduce(allreduce, 10000);
106 }
107
108 TEST(IallreduceReplacement, ReductionFactor5)
109 {
110 stk::impl::IAllreduceReplacement<int> allreduce(MPI_COMM_WORLD, MPI_INT, 666, 5);
111 MPI_Barrier(MPI_COMM_WORLD);
112 test_allreduce(allreduce, 1);
113 test_allreduce(allreduce, 1);
114 test_allreduce(allreduce, 2);
115 test_allreduce(allreduce, 10);
116 test_allreduce(allreduce, 100);
117 test_allreduce(allreduce, 1000);
118 test_allreduce(allreduce, 10000);
119 }
120
121 TEST(IallreduceReplacement, ReductionFactorError)
122 {
123 EXPECT_ANY_THROW(stk::impl::IAllreduceReplacement<int> allreduce(MPI_COMM_WORLD, MPI_INT, 666, 1));
124 }
125
126 TEST(IallreduceReplacement, InputSizeError)
127 {
128 auto op = [](const int& lhs, const int& rhs) { return lhs + rhs; };
129 stk::impl::IAllreduceReplacement<int> allreduce(MPI_COMM_WORLD, MPI_INT, 666, 2);
130 std::vector<int> input_data(2), output_data(3);
131 EXPECT_ANY_THROW(allreduce.startReduction(op, input_data, output_data));
132 }
133
134 TEST(IallreduceReplacement, ReuseBeforeCompletedError)
135 {
136 if (stk::parallel_machine_size(MPI_COMM_WORLD) < 4)
137 {
138 GTEST_SKIP();
139 }
140
141 auto op = [](const int& lhs, const int& rhs) { return lhs + rhs; };
142 stk::impl::IAllreduceReplacement<int> allreduce(MPI_COMM_WORLD, MPI_INT, 666, 2);
143 std::vector<int> input_data(2, 0), output_data(2, 0);
144 allreduce.startReduction(op, input_data, output_data);
145
146 EXPECT_ANY_THROW(allreduce.startReduction(op, input_data, output_data));
147
148 allreduce.finishReduction();
149 }