| 1 |
/* |
| 2 |
* (C) Crown Copyright 2021 Met Office |
| 3 |
* |
| 4 |
* This software is licensed under the terms of the Apache Licence Version 2.0 |
| 5 |
* which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. |
| 6 |
*/ |
| 7 |
|
| 8 |
#include "RedistributeStructuredColumns.h" |
| 9 |
|
| 10 |
#include <algorithm> |
| 11 |
#include <numeric> |
| 12 |
|
| 13 |
#include "atlas/array/MakeView.h" |
| 14 |
#include "atlas/field.h" |
| 15 |
#include "atlas/field/FieldSet.h" |
| 16 |
#include "atlas/functionspace/StructuredColumns.h" |
| 17 |
#include "atlas/parallel/mpi/mpi.h" |
| 18 |
#include "atlas/runtime/Exception.h" |
| 19 |
|
| 20 |
namespace atlas { |
| 21 |
namespace redistribution { |
| 22 |
namespace detail { |
| 23 |
|
| 24 |
// Anonymous name space for helper functions. |
| 25 |
namespace { |
| 26 |
|
| 27 |
// Transform a vector element-by-element with a functor. |
| 28 |
template <typename outType, typename inType, typename functorType> |
| 29 |
std::vector<outType> transformVector(const std::vector<inType>& inVector, const functorType& functor) { |
| 30 |
// Declare result. |
| 31 |
auto outVector = std::vector<outType>{}; |
| 32 |
outVector.reserve(inVector.size()); |
| 33 |
|
| 34 |
// Transform vector. |
| 35 |
std::transform(inVector.begin(), inVector.end(), std::back_inserter(outVector), functor); |
| 36 |
|
| 37 |
return outVector; |
| 38 |
} |
| 39 |
|
| 40 |
// Visit each index in a StructuredIndexRangeVector. |
| 41 |
template <typename functorType> |
| 42 |
void forEachIndex(const StructuredIndexRangeVector& ranges, const functorType& functor) { |
| 43 |
// Loop over all ranges. |
| 44 |
std::for_each(ranges.cbegin(), ranges.cend(), [&](const StructuredIndexRange& range) { |
| 45 |
range.forEach(functor); |
| 46 |
return; |
| 47 |
}); |
| 48 |
return; |
| 49 |
} |
| 50 |
|
| 51 |
} // namespace |
| 52 |
|
| 53 |
//======================================================================== |
| 54 |
// Class public methods implementation. |
| 55 |
//======================================================================== |
| 56 |
|
| 57 |
// Constructor. |
| 58 |
void RedistributeStructuredColumns::do_setup() { |
| 59 |
source_ = source(); |
| 60 |
target_ = target(); |
| 61 |
|
| 62 |
// Assert that functionspaces are StructuredColumns. |
| 63 |
ATLAS_ASSERT(source_, source().type() + " must be StructuredColumns"); |
| 64 |
ATLAS_ASSERT(target_, target().type() + " must be StructuredColumns"); |
| 65 |
|
| 66 |
// Check that grids match. |
| 67 |
ATLAS_ASSERT(source_.grid().name() == target_.grid().name()); |
| 68 |
|
| 69 |
// Check levels match. |
| 70 |
ATLAS_ASSERT(source_.levels() == target_.levels()); |
| 71 |
|
| 72 |
// Check that communicators match. |
| 73 |
ATLAS_ASSERT(source_.mpi_comm() == target_.mpi_comm()); |
| 74 |
mpi_comm_ = source_.mpi_comm(); |
| 75 |
|
| 76 |
|
| 77 |
// Get source and target range of this function space. |
| 78 |
const auto sourceRange = StructuredIndexRange(source_); |
| 79 |
const auto targetRange = StructuredIndexRange(target_); |
| 80 |
|
| 81 |
|
| 82 |
// Get source and target ranges over all PEs. |
| 83 |
const auto sourceRanges = sourceRange.getStructuredIndexRanges(); |
| 84 |
const auto targetRanges = targetRange.getStructuredIndexRanges(); |
| 85 |
|
| 86 |
|
| 87 |
// Get intersections between sourceRange and targetRanges. |
| 88 |
auto getIntersections = [](const StructuredIndexRange& range, const StructuredIndexRangeVector& ranges) { |
| 89 |
return transformVector<StructuredIndexRange>( |
| 90 |
ranges, [&](const StructuredIndexRange& rangesElem) { return range & rangesElem; }); |
| 91 |
}; |
| 92 |
|
| 93 |
sendIntersections_ = getIntersections(sourceRange, targetRanges); |
| 94 |
recvIntersections_ = getIntersections(targetRange, sourceRanges); |
| 95 |
|
| 96 |
|
| 97 |
// Get counts and displacements for MPI communication. |
| 98 |
auto getCountsDisplacements = [](const StructuredIndexRangeVector& intersections, const idx_t levels) { |
| 99 |
const auto counts = transformVector<int>(intersections, [&](const StructuredIndexRange& intersection) { |
| 100 |
return static_cast<int>(intersection.getElemCount() * levels); |
| 101 |
}); |
| 102 |
|
| 103 |
auto displacements = std::vector<int>{0}; |
| 104 |
std::partial_sum(counts.cbegin(), counts.cend() - 1, std::back_inserter(displacements)); |
| 105 |
|
| 106 |
return std::make_pair(counts, displacements); |
| 107 |
}; |
| 108 |
|
| 109 |
std::tie(sendCounts_, sendDisplacements_) = getCountsDisplacements(sendIntersections_, source_.levels()); |
| 110 |
|
| 111 |
std::tie(recvCounts_, recvDisplacements_) = getCountsDisplacements(recvIntersections_, target_.levels()); |
| 112 |
|
| 113 |
|
| 114 |
// Trim off invalid intersections. |
| 115 |
auto trimIntersections = [](StructuredIndexRangeVector& intersections) { |
| 116 |
intersections.erase( |
| 117 |
std::remove_if(intersections.begin(), intersections.end(), |
| 118 |
[](const StructuredIndexRange& intersection) { return !(intersection.getElemCount()); }), |
| 119 |
intersections.end()); |
| 120 |
|
| 121 |
return; |
| 122 |
}; |
| 123 |
|
| 124 |
trimIntersections(sendIntersections_); |
| 125 |
trimIntersections(recvIntersections_); |
| 126 |
|
| 127 |
return; |
| 128 |
} |
| 129 |
|
| 130 |
void RedistributeStructuredColumns::execute(const Field& sourceField, Field& targetField) const { |
| 131 |
// Assert that fields are defined on StructuredColumns. |
| 132 |
ATLAS_ASSERT(functionspace::StructuredColumns(sourceField.functionspace())); |
| 133 |
ATLAS_ASSERT(functionspace::StructuredColumns(targetField.functionspace())); |
| 134 |
|
| 135 |
// Check that grids match. |
| 136 |
ATLAS_ASSERT(functionspace::StructuredColumns(sourceField.functionspace()).grid().name() == source_.grid().name()); |
| 137 |
ATLAS_ASSERT(functionspace::StructuredColumns(targetField.functionspace()).grid().name() == target_.grid().name()); |
| 138 |
|
| 139 |
// Check levels match. |
| 140 |
ATLAS_ASSERT(sourceField.levels() == source_.levels()); |
| 141 |
ATLAS_ASSERT(targetField.levels() == target_.levels()); |
| 142 |
|
| 143 |
// Determine data type of field and execute. |
| 144 |
switch (sourceField.datatype().kind()) { |
| 145 |
case array::DataType::KIND_REAL64: |
| 146 |
do_execute<double>(sourceField, targetField); |
| 147 |
break; |
| 148 |
|
| 149 |
case array::DataType::KIND_REAL32: |
| 150 |
do_execute<float>(sourceField, targetField); |
| 151 |
break; |
| 152 |
|
| 153 |
case array::DataType::KIND_INT32: |
| 154 |
do_execute<int>(sourceField, targetField); |
| 155 |
break; |
| 156 |
|
| 157 |
case array::DataType::KIND_INT64: |
| 158 |
do_execute<long>(sourceField, targetField); |
| 159 |
break; |
| 160 |
|
| 161 |
default: |
| 162 |
throw_NotImplemented("No implementation for data type " + sourceField.datatype().str(), Here()); |
| 163 |
} |
| 164 |
|
| 165 |
return; |
| 166 |
} |
| 167 |
|
| 168 |
void RedistributeStructuredColumns::execute(const FieldSet& sourceFieldSet, FieldSet& targetFieldSet) const { |
| 169 |
// Check that both FieldSets are the same size. |
| 170 |
ATLAS_ASSERT(sourceFieldSet.size() == targetFieldSet.size()); |
| 171 |
|
| 172 |
auto targetFieldSetIt = targetFieldSet.begin(); |
| 173 |
std::for_each(sourceFieldSet.cbegin(), sourceFieldSet.cend(), [&](const Field& sourceField) { |
| 174 |
execute(sourceField, *targetFieldSetIt++); |
| 175 |
return; |
| 176 |
}); |
| 177 |
|
| 178 |
return; |
| 179 |
} |
| 180 |
|
| 181 |
//======================================================================== |
| 182 |
// Class private methods implementation. |
| 183 |
//======================================================================== |
| 184 |
|
| 185 |
template <typename fieldType> |
| 186 |
void RedistributeStructuredColumns::do_execute(const Field& sourceField, Field& targetField) const { |
| 187 |
// Make Atlas view objects. |
| 188 |
const auto sourceView = array::make_view<fieldType, 2>(sourceField); |
| 189 |
auto targetView = array::make_view<fieldType, 2>(targetField); |
| 190 |
|
| 191 |
|
| 192 |
// Get buffer sizes. |
| 193 |
const auto nSend = sendCounts_.back() + sendDisplacements_.back(); |
| 194 |
const auto nRecv = recvCounts_.back() + recvDisplacements_.back(); |
| 195 |
|
| 196 |
|
| 197 |
// Allocate send and receive buffers. |
| 198 |
auto sendBuffer = std::vector<fieldType>(static_cast<size_t>(nSend)); |
| 199 |
auto recvBuffer = std::vector<fieldType>(static_cast<size_t>(nRecv)); |
| 200 |
|
| 201 |
|
| 202 |
// Set send functor. |
| 203 |
auto sendBufferIt = sendBuffer.begin(); |
| 204 |
auto sendFunctor = [&](const idx_t i, const idx_t j) { |
| 205 |
// Loop over levels |
| 206 |
const auto iNode = source_.index(i, j); |
| 207 |
const auto kEnd = source_.levels(); |
| 208 |
for (idx_t k = 0; k < kEnd; ++k) { |
| 209 |
*sendBufferIt++ = sourceView(iNode, k); |
| 210 |
} |
| 211 |
return; |
| 212 |
}; |
| 213 |
|
| 214 |
|
| 215 |
// Set receive functor. |
| 216 |
auto recvBufferIt = recvBuffer.cbegin(); |
| 217 |
auto recvFunctor = [&](const idx_t i, const idx_t j) { |
| 218 |
// Loop over levels |
| 219 |
const auto iNode = target_.index(i, j); |
| 220 |
const auto kEnd = target_.levels(); |
| 221 |
for (idx_t k = 0; k < kEnd; ++k) { |
| 222 |
targetView(iNode, k) = *recvBufferIt++; |
| 223 |
} |
| 224 |
return; |
| 225 |
}; |
| 226 |
|
| 227 |
|
| 228 |
// Write data to buffer. |
| 229 |
forEachIndex(sendIntersections_, sendFunctor); |
| 230 |
|
| 231 |
// Communicate. |
| 232 |
mpi::comm(mpi_comm_).allToAllv(sendBuffer.data(), sendCounts_.data(), sendDisplacements_.data(), recvBuffer.data(), |
| 233 |
recvCounts_.data(), recvDisplacements_.data()); |
| 234 |
|
| 235 |
// Read data from buffer. |
| 236 |
forEachIndex(recvIntersections_, recvFunctor); |
| 237 |
|
| 238 |
return; |
| 239 |
} |
| 240 |
|
| 241 |
//======================================================================== |
| 242 |
// Index range methods. |
| 243 |
//======================================================================== |
| 244 |
|
| 245 |
// Constructor. |
| 246 |
StructuredIndexRange::StructuredIndexRange(const functionspace::StructuredColumns& structuredColumns) { |
| 247 |
jBeginEnd_ = std::make_pair(structuredColumns.j_begin(), structuredColumns.j_end()); |
| 248 |
|
| 249 |
for (auto j = jBeginEnd_.first; j < jBeginEnd_.second; ++j) { |
| 250 |
iBeginEnd_.emplace_back(structuredColumns.i_begin(j), structuredColumns.i_end(j)); |
| 251 |
} |
| 252 |
|
| 253 |
mpi_comm_ = structuredColumns.mpi_comm(); |
| 254 |
|
| 255 |
return; |
| 256 |
} |
| 257 |
|
| 258 |
// Get index ranges from all PEs. |
| 259 |
StructuredIndexRangeVector StructuredIndexRange::getStructuredIndexRanges() const { |
| 260 |
auto& comm = mpi::comm(mpi_comm()); |
| 261 |
|
| 262 |
// Get MPI communicator size. |
| 263 |
const auto mpiSize = static_cast<size_t>(comm.size()); |
| 264 |
|
| 265 |
// Set recv buffer for j range. |
| 266 |
auto jRecvBuffer = idxPairVector(mpiSize); |
| 267 |
|
| 268 |
// Perform all gather. |
| 269 |
comm.allGather(jBeginEnd_, jRecvBuffer.begin(), jRecvBuffer.end()); |
| 270 |
|
| 271 |
// Set i receive counts. |
| 272 |
auto iRecvCounts = transformVector<int>( |
| 273 |
jRecvBuffer, [](const idxPair& jElem) { return static_cast<int>(jElem.second - jElem.first); }); |
| 274 |
|
| 275 |
// Set recv displacements for i range. |
| 276 |
auto iRecvDisplacements = std::vector<int>{0}; |
| 277 |
std::partial_sum(iRecvCounts.cbegin(), iRecvCounts.cend() - 1, std::back_inserter(iRecvDisplacements)); |
| 278 |
|
| 279 |
// Set recv buffer for i range. |
| 280 |
auto irecvBuffer = idxPairVector(static_cast<size_t>(iRecvDisplacements.back() + iRecvCounts.back())); |
| 281 |
|
| 282 |
// Perform all gather. |
| 283 |
comm.allGatherv(iBeginEnd_.cbegin(), iBeginEnd_.cend(), irecvBuffer.begin(), iRecvCounts.data(), |
| 284 |
iRecvDisplacements.data()); |
| 285 |
|
| 286 |
// Make vector of indexRange structs. |
| 287 |
auto indexRanges = StructuredIndexRangeVector{}; |
| 288 |
for (size_t i = 0; i < mpiSize; ++i) { |
| 289 |
auto indexRange = StructuredIndexRange{}; |
| 290 |
indexRange.jBeginEnd_ = jRecvBuffer[i]; |
| 291 |
const auto iBegin = irecvBuffer.cbegin() + iRecvDisplacements[i]; |
| 292 |
const auto iEnd = iBegin + iRecvCounts[i]; |
| 293 |
std::copy(iBegin, iEnd, std::back_inserter(indexRange.iBeginEnd_)); |
| 294 |
|
| 295 |
indexRanges.push_back(indexRange); |
| 296 |
} |
| 297 |
|
| 298 |
return indexRanges; |
| 299 |
} |
| 300 |
|
| 301 |
// Count number of elements in index range. |
| 302 |
idx_t StructuredIndexRange::getElemCount() const { |
| 303 |
// Accumulate size of positive i range. |
| 304 |
const auto count = |
| 305 |
std::accumulate(iBeginEnd_.cbegin(), iBeginEnd_.cend(), 0, [](const idx_t cumulant, const idxPair iElem) { |
| 306 |
// Only count positive differences. |
| 307 |
return cumulant + static_cast<idx_t>(std::max(iElem.second - iElem.first, idx_t(0))); |
| 308 |
}); |
| 309 |
|
| 310 |
return count; |
| 311 |
} |
| 312 |
|
| 313 |
// Return the intersection between two index ranges. |
| 314 |
StructuredIndexRange StructuredIndexRange::operator&(const StructuredIndexRange& indexRange) const { |
| 315 |
// Declare result. |
| 316 |
auto intersection = StructuredIndexRange{}; |
| 317 |
|
| 318 |
// get j intersection range. |
| 319 |
intersection.jBeginEnd_ = std::make_pair(std::max(jBeginEnd_.first, indexRange.jBeginEnd_.first), |
| 320 |
std::min(jBeginEnd_.second, indexRange.jBeginEnd_.second)); |
| 321 |
|
| 322 |
// get i intersection range. |
| 323 |
if (intersection.jBeginEnd_.first < intersection.jBeginEnd_.second) { |
| 324 |
// get iterators. |
| 325 |
const auto iBeginA = iBeginEnd_.cbegin() + intersection.jBeginEnd_.first - jBeginEnd_.first; |
| 326 |
const auto iBeginB = |
| 327 |
indexRange.iBeginEnd_.cbegin() + intersection.jBeginEnd_.first - indexRange.jBeginEnd_.first; |
| 328 |
const auto iEndA = iBeginEnd_.cend() + intersection.jBeginEnd_.second - jBeginEnd_.second; |
| 329 |
|
| 330 |
std::transform(iBeginA, iEndA, iBeginB, std::back_inserter(intersection.iBeginEnd_), |
| 331 |
[](const idxPair iElemA, const idxPair iElemB) { |
| 332 |
return std::make_pair(std::max(iElemA.first, iElemB.first), |
| 333 |
std::min(iElemA.second, iElemB.second)); |
| 334 |
}); |
| 335 |
} |
| 336 |
return intersection; |
| 337 |
} |
| 338 |
|
| 339 |
// Loop over all indices. Functor should have signature |
| 340 |
// functor(const idx_t i, const idx_t j). |
| 341 |
template <typename functorType> |
| 342 |
void StructuredIndexRange::forEach(const functorType& functor) const { |
| 343 |
auto iBeginEndIt = iBeginEnd_.begin(); |
| 344 |
|
| 345 |
// Loop over j. |
| 346 |
for (auto j = jBeginEnd_.first; j < jBeginEnd_.second; ++j) { |
| 347 |
const auto iBeginEnd = *iBeginEndIt++; |
| 348 |
|
| 349 |
// Loop over i. |
| 350 |
for (auto i = iBeginEnd.first; i < iBeginEnd.second; ++i) { |
| 351 |
// Call functor. |
| 352 |
functor(i, j); |
| 353 |
} |
| 354 |
} |
| 355 |
|
| 356 |
return; |
| 357 |
} |
| 358 |
|
| 359 |
namespace { |
| 360 |
static RedistributionImplBuilder<RedistributeStructuredColumns> register_builder( |
| 361 |
RedistributeStructuredColumns::static_type()); |
| 362 |
} |
| 363 |
|
| 364 |
} // namespace detail |
| 365 |
} // namespace redistribution |
| 366 |
} // namespace atlas |