| 1 |
// Copyright 2002 - 2008, 2010, 2011 National Technology Engineering |
| 2 |
// Solutions of Sandia, LLC (NTESS). Under the terms of Contract |
| 3 |
// DE-NA0003525 with NTESS, the U.S. Government retains certain rights |
| 4 |
// in this software. |
| 5 |
// |
| 6 |
// Redistribution and use in source and binary forms, with or without |
| 7 |
// modification, are permitted provided that the following conditions are |
| 8 |
// met: |
| 9 |
// |
| 10 |
// * Redistributions of source code must retain the above copyright |
| 11 |
// notice, this list of conditions and the following disclaimer. |
| 12 |
// |
| 13 |
// * Redistributions in binary form must reproduce the above |
| 14 |
// copyright notice, this list of conditions and the following |
| 15 |
// disclaimer in the documentation and/or other materials provided |
| 16 |
// with the distribution. |
| 17 |
// |
| 18 |
// * Neither the name of NTESS nor the names of its contributors |
| 19 |
// may be used to endorse or promote products derived from this |
| 20 |
// software without specific prior written permission. |
| 21 |
// |
| 22 |
// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS |
| 23 |
// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT |
| 24 |
// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR |
| 25 |
// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT |
| 26 |
// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, |
| 27 |
// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT |
| 28 |
// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, |
| 29 |
// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY |
| 30 |
// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT |
| 31 |
// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE |
| 32 |
// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. |
| 33 |
// |
| 34 |
|
| 35 |
#include <string> |
| 36 |
#include <gtest/gtest.h> |
| 37 |
#include <stk_util/parallel/Parallel.hpp> |
| 38 |
#include <stk_util/parallel/ParallelComm.hpp> |
| 39 |
#include <stk_unit_test_utils/timer.hpp> |
| 40 |
#include <stk_unit_test_utils/getOption.h> |
| 41 |
#include <vector> |
| 42 |
#include <iostream> |
| 43 |
#include <chrono> |
| 44 |
#include <thread> |
| 45 |
|
| 46 |
void fill_comm_data(int numProcs, int myProc, |
| 47 |
unsigned BANDWIDTH, unsigned NUM_VALUES, |
| 48 |
std::vector<int>& sendOffsets, |
| 49 |
std::vector<double>& sendData, |
| 50 |
std::vector<int>& recvOffsets, |
| 51 |
std::vector<double>& recvData) |
| 52 |
{ |
| 53 |
sendOffsets.assign(numProcs+1, 0); |
| 54 |
|
| 55 |
const int halfBandwidth = BANDWIDTH/2; |
| 56 |
const int rawFirstDestProc = myProc - halfBandwidth; |
| 57 |
const int rawLastDestProc = myProc + halfBandwidth; |
| 58 |
int firstDestProc = std::max(0, rawFirstDestProc); |
| 59 |
if (firstDestProc == myProc) { |
| 60 |
firstDestProc += 1; |
| 61 |
} |
| 62 |
|
| 63 |
int lastDestProc = std::min(numProcs-1, rawLastDestProc); |
| 64 |
if (lastDestProc == myProc) { |
| 65 |
lastDestProc -= 1; |
| 66 |
} |
| 67 |
|
| 68 |
//first, put send-sizes in sendOffsets: |
| 69 |
for(int p=firstDestProc; p<=lastDestProc; ++p) { |
| 70 |
if (p != myProc) { |
| 71 |
sendOffsets[p] = NUM_VALUES; |
| 72 |
} |
| 73 |
} |
| 74 |
|
| 75 |
recvOffsets = sendOffsets; |
| 76 |
|
| 77 |
//the send/recv procs are 'symmetric', but data-sizes will be non-symmetric: |
| 78 |
//add some variation to the send-sizes |
| 79 |
for(int p=firstDestProc; p<=lastDestProc; ++p) { |
| 80 |
if (p != myProc) { |
| 81 |
sendOffsets[p] += p; |
| 82 |
recvOffsets[p] += myProc; |
| 83 |
} |
| 84 |
} |
| 85 |
|
| 86 |
//now, convert sizes to offsets: |
| 87 |
unsigned sendOffset = 0; |
| 88 |
unsigned recvOffset = 0; |
| 89 |
for(unsigned p=0; p<sendOffsets.size(); ++p) { |
| 90 |
unsigned size = sendOffsets[p]; |
| 91 |
sendOffsets[p] = sendOffset; |
| 92 |
sendOffset += size; |
| 93 |
|
| 94 |
size = recvOffsets[p]; |
| 95 |
recvOffsets[p] = recvOffset; |
| 96 |
recvOffset += size; |
| 97 |
} |
| 98 |
|
| 99 |
sendData.assign(sendOffset, 42.0); |
| 100 |
recvData.assign(recvOffset, 0); |
| 101 |
} |
| 102 |
|
| 103 |
TEST(PerfPllDataExchange, nonsym_knownsizes) |
| 104 |
{ |
| 105 |
stk::ParallelMachine comm = MPI_COMM_WORLD; |
| 106 |
const int numProcs = stk::parallel_machine_size(comm); |
| 107 |
if (numProcs < 3) { GTEST_SKIP(); } |
| 108 |
|
| 109 |
const unsigned NUM_BATCHES = 5; |
| 110 |
const unsigned NUM_RUNS = stk::unit_test_util::get_command_line_option("-r", 5); |
| 111 |
const unsigned defaultBandwidth = std::max(2, numProcs/5); |
| 112 |
const unsigned BANDWIDTH = stk::unit_test_util::get_command_line_option("-b", defaultBandwidth); |
| 113 |
const unsigned NUM_VALUES = stk::unit_test_util::get_command_line_option("-v", 20); |
| 114 |
|
| 115 |
const int myProc = stk::parallel_machine_rank(comm); |
| 116 |
|
| 117 |
if (myProc == 0) { |
| 118 |
std::cout << "Using NUM_RUNS=" << NUM_RUNS |
| 119 |
<< ", BANDWIDTH=" << BANDWIDTH |
| 120 |
<< ", NUM_VALUES=" << NUM_VALUES |
| 121 |
<< std::endl; |
| 122 |
} |
| 123 |
|
| 124 |
stk::unit_test_util::BatchTimer batchTimer(comm); |
| 125 |
batchTimer.initialize_batch_timer(); |
| 126 |
|
| 127 |
std::vector<int> sendOffsets, recvOffsets; |
| 128 |
std::vector<double> sendData, recvData; |
| 129 |
|
| 130 |
fill_comm_data(numProcs, myProc, BANDWIDTH, NUM_VALUES, sendOffsets, sendData, recvOffsets, recvData); |
| 131 |
|
| 132 |
for(unsigned b=0; b<NUM_BATCHES; ++b) { |
| 133 |
batchTimer.start_batch_timer(); |
| 134 |
|
| 135 |
for(unsigned r=0; r<NUM_RUNS; ++r) { |
| 136 |
|
| 137 |
constexpr bool checkInput = false; |
| 138 |
stk::parallel_data_exchange_nonsym_known_sizes_t(sendOffsets.data(), sendData.data(), |
| 139 |
recvOffsets.data(), recvData.data(), |
| 140 |
comm, checkInput); |
| 141 |
} |
| 142 |
|
| 143 |
batchTimer.stop_batch_timer(); |
| 144 |
} |
| 145 |
|
| 146 |
batchTimer.print_batch_timing(NUM_RUNS); |
| 147 |
} |
| 148 |
|