| 1 |
#include "gtest/gtest.h" |
| 2 |
#include "stk_util/parallel/IallreduceReplacement.hpp" |
| 3 |
#include "stk_util/parallel/Parallel.hpp" |
| 4 |
|
| 5 |
namespace |
| 6 |
{ |
| 7 |
|
| 8 |
void sum_op(void* invec, void* inoutvec, int* len, MPI_Datatype* /*datatype*/) |
| 9 |
{ |
| 10 |
int* invec_int = reinterpret_cast<int*>(invec); |
| 11 |
int* inoutvec_int = reinterpret_cast<int*>(inoutvec); |
| 12 |
for (int i=0; i < *len; ++i) |
| 13 |
inoutvec_int[i] = invec_int[i] + inoutvec_int[i]; |
| 14 |
} |
| 15 |
|
| 16 |
double test_allreduce_mpi(int numValues) |
| 17 |
{ |
| 18 |
int num_iters = 100; |
| 19 |
double t_start = 0; |
| 20 |
|
| 21 |
std::vector<int> input_data(numValues, 1), output_data(numValues, 42); |
| 22 |
MPI_Op mpi_op; |
| 23 |
MPI_Op_create(&sum_op, true, &mpi_op); |
| 24 |
for (int i=0; i < num_iters; ++i) |
| 25 |
{ |
| 26 |
if (i == 1) |
| 27 |
t_start = MPI_Wtime(); |
| 28 |
|
| 29 |
MPI_Request req; |
| 30 |
//MPI_Ibarrier(MPI_COMM_WORLD, &req); |
| 31 |
MPI_Iallreduce(input_data.data(), output_data.data(), numValues, MPI_INT, mpi_op, MPI_COMM_WORLD, &req); |
| 32 |
MPI_Wait(&req, MPI_STATUS_IGNORE); |
| 33 |
} |
| 34 |
|
| 35 |
double elapsed_time = (MPI_Wtime() - t_start) / (num_iters - 1); |
| 36 |
double max_time = 0; |
| 37 |
MPI_Allreduce(&elapsed_time, &max_time, 1, MPI_DOUBLE, MPI_MAX, MPI_COMM_WORLD); |
| 38 |
|
| 39 |
return max_time; |
| 40 |
} |
| 41 |
|
| 42 |
|
| 43 |
void test_allreduce(stk::impl::IAllreduceReplacement<int>& allreduce, int numValues) |
| 44 |
{ |
| 45 |
auto op = [](const int& lhs, const int& rhs) { return lhs + rhs; }; |
| 46 |
std::vector<int> input_data(numValues, stk::parallel_machine_rank(MPI_COMM_WORLD)), output_data(numValues, 42), |
| 47 |
expected_data(numValues); |
| 48 |
|
| 49 |
int commRank = stk::parallel_machine_rank(MPI_COMM_WORLD); |
| 50 |
int commSize = stk::parallel_machine_size(MPI_COMM_WORLD); |
| 51 |
|
| 52 |
if (commRank == 0) |
| 53 |
std::cout << "\nTesting size " << numValues << std::endl; |
| 54 |
|
| 55 |
|
| 56 |
for (int i=0; i < numValues; ++i) |
| 57 |
{ |
| 58 |
input_data[i] = commRank + i; |
| 59 |
expected_data[i] = i*commSize + commSize * (commSize - 1) / 2; |
| 60 |
} |
| 61 |
|
| 62 |
int num_iters = 100; |
| 63 |
double t_start = 0; |
| 64 |
|
| 65 |
for (int i=0; i < num_iters; ++i) |
| 66 |
{ |
| 67 |
if (i == 1) |
| 68 |
t_start = MPI_Wtime(); |
| 69 |
|
| 70 |
allreduce.startReduction(op, input_data, output_data); |
| 71 |
allreduce.finishReduction(); |
| 72 |
} |
| 73 |
|
| 74 |
double elapsed_time = (MPI_Wtime() - t_start) / (num_iters - 1); |
| 75 |
double max_time = 0; |
| 76 |
MPI_Allreduce(&elapsed_time, &max_time, 1, MPI_DOUBLE, MPI_MAX, MPI_COMM_WORLD); |
| 77 |
if (commRank == 0) |
| 78 |
std::cout << "New algorithm time = " << max_time << " per iteration" << std::endl; |
| 79 |
|
| 80 |
for (int i=0; i < numValues; ++i) |
| 81 |
{ |
| 82 |
EXPECT_EQ(output_data[i], expected_data[i]); |
| 83 |
} |
| 84 |
|
| 85 |
double max_time2 = test_allreduce_mpi(numValues); |
| 86 |
|
| 87 |
if (commRank == 0) |
| 88 |
std::cout << "MPI algorithm time = " << max_time2 << " per iteration" << ", ratio " << max_time / max_time2 << std::endl; |
| 89 |
} |
| 90 |
} |
| 91 |
|
| 92 |
|
| 93 |
|
| 94 |
|
| 95 |
TEST(IallreduceReplacement, ReductionFactor2) |
| 96 |
{ |
| 97 |
stk::impl::IAllreduceReplacement<int> allreduce(MPI_COMM_WORLD, MPI_INT, 666, 2); |
| 98 |
MPI_Barrier(MPI_COMM_WORLD); |
| 99 |
test_allreduce(allreduce, 1); |
| 100 |
test_allreduce(allreduce, 1); |
| 101 |
test_allreduce(allreduce, 2); |
| 102 |
test_allreduce(allreduce, 10); |
| 103 |
test_allreduce(allreduce, 100); |
| 104 |
test_allreduce(allreduce, 1000); |
| 105 |
test_allreduce(allreduce, 10000); |
| 106 |
} |
| 107 |
|
| 108 |
TEST(IallreduceReplacement, ReductionFactor5) |
| 109 |
{ |
| 110 |
stk::impl::IAllreduceReplacement<int> allreduce(MPI_COMM_WORLD, MPI_INT, 666, 5); |
| 111 |
MPI_Barrier(MPI_COMM_WORLD); |
| 112 |
test_allreduce(allreduce, 1); |
| 113 |
test_allreduce(allreduce, 1); |
| 114 |
test_allreduce(allreduce, 2); |
| 115 |
test_allreduce(allreduce, 10); |
| 116 |
test_allreduce(allreduce, 100); |
| 117 |
test_allreduce(allreduce, 1000); |
| 118 |
test_allreduce(allreduce, 10000); |
| 119 |
} |
| 120 |
|
| 121 |
TEST(IallreduceReplacement, ReductionFactorError) |
| 122 |
{ |
| 123 |
EXPECT_ANY_THROW(stk::impl::IAllreduceReplacement<int> allreduce(MPI_COMM_WORLD, MPI_INT, 666, 1)); |
| 124 |
} |
| 125 |
|
| 126 |
TEST(IallreduceReplacement, InputSizeError) |
| 127 |
{ |
| 128 |
auto op = [](const int& lhs, const int& rhs) { return lhs + rhs; }; |
| 129 |
stk::impl::IAllreduceReplacement<int> allreduce(MPI_COMM_WORLD, MPI_INT, 666, 2); |
| 130 |
std::vector<int> input_data(2), output_data(3); |
| 131 |
EXPECT_ANY_THROW(allreduce.startReduction(op, input_data, output_data)); |
| 132 |
} |
| 133 |
|
| 134 |
TEST(IallreduceReplacement, ReuseBeforeCompletedError) |
| 135 |
{ |
| 136 |
if (stk::parallel_machine_size(MPI_COMM_WORLD) < 4) |
| 137 |
{ |
| 138 |
GTEST_SKIP(); |
| 139 |
} |
| 140 |
|
| 141 |
auto op = [](const int& lhs, const int& rhs) { return lhs + rhs; }; |
| 142 |
stk::impl::IAllreduceReplacement<int> allreduce(MPI_COMM_WORLD, MPI_INT, 666, 2); |
| 143 |
std::vector<int> input_data(2, 0), output_data(2, 0); |
| 144 |
allreduce.startReduction(op, input_data, output_data); |
| 145 |
|
| 146 |
EXPECT_ANY_THROW(allreduce.startReduction(op, input_data, output_data)); |
| 147 |
|
| 148 |
allreduce.finishReduction(); |
| 149 |
} |