| 1 |
/* |
| 2 |
* (C) Crown Copyright 2021 Met Office |
| 3 |
* |
| 4 |
* This software is licensed under the terms of the Apache Licence Version 2.0 |
| 5 |
* which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. |
| 6 |
*/ |
| 7 |
|
| 8 |
#include <numeric> |
| 9 |
#include <vector> |
| 10 |
|
| 11 |
#include "atlas/field/Field.h" |
| 12 |
#include "atlas/functionspace/CellColumns.h" |
| 13 |
#include "atlas/functionspace/EdgeColumns.h" |
| 14 |
#include "atlas/functionspace/NodeColumns.h" |
| 15 |
#include "atlas/functionspace/PointCloud.h" |
| 16 |
#include "atlas/functionspace/StructuredColumns.h" |
| 17 |
#include "atlas/mesh/Mesh.h" |
| 18 |
#include "atlas/parallel/mpi/mpi.h" |
| 19 |
#include "atlas/redistribution/detail/RedistributeGeneric.h" |
| 20 |
#include "atlas/redistribution/detail/RedistributionImplFactory.h" |
| 21 |
#include "atlas/util/Unique.h" |
| 22 |
|
| 23 |
|
| 24 |
namespace atlas { |
| 25 |
namespace redistribution { |
| 26 |
namespace detail { |
| 27 |
|
| 28 |
using mesh::HybridElements; |
| 29 |
using mesh::Nodes; |
| 30 |
|
| 31 |
// Helper type definitions and functions for redistribution. |
| 32 |
namespace { |
| 33 |
|
| 34 |
// Define index-UID struct. (Needed to overload "<"). |
| 35 |
struct IdxUid : public std::pair<idx_t, uidx_t> { |
| 36 |
using std::pair<idx_t, uidx_t>::pair; |
| 37 |
}; |
| 38 |
|
| 39 |
Field getGhostField(const FunctionSpace& functionspace) { |
| 40 |
if (functionspace::NodeColumns(functionspace) || functionspace::StructuredColumns(functionspace) || |
| 41 |
functionspace::PointCloud(functionspace)) { |
| 42 |
return functionspace.ghost(); |
| 43 |
} |
| 44 |
if (functionspace::EdgeColumns(functionspace) || functionspace::CellColumns(functionspace)) { |
| 45 |
// TODO: Move something like this into the functionspace::EdgeColumns and functionspace::CellColumns |
| 46 |
auto& comm = mpi::comm(functionspace.mpi_comm()); |
| 47 |
|
| 48 |
|
| 49 |
// Get mesh elements. |
| 50 |
Mesh mesh = functionspace::EdgeColumns(functionspace) |
| 51 |
? functionspace::EdgeColumns(functionspace).mesh() |
| 52 |
: functionspace::CellColumns(functionspace).mesh(); |
| 53 |
const auto& elems = functionspace::EdgeColumns(functionspace) |
| 54 |
? mesh.edges() |
| 55 |
: mesh.cells(); |
| 56 |
|
| 57 |
// Make ghost field. |
| 58 |
auto ghost_field = Field("ghost", array::make_datatype<int>(), array::make_shape(functionspace.size())); |
| 59 |
|
| 60 |
// Make views. |
| 61 |
auto ghost = array::make_view<int, 1>(ghost_field); |
| 62 |
auto remote_index = array::make_indexview<idx_t, 1>(elems.remote_index()); |
| 63 |
auto partition = array::make_view<int, 1>(elems.partition()); |
| 64 |
|
| 65 |
// Set ghost field. |
| 66 |
const auto thisPart = static_cast<int>(comm.rank()); |
| 67 |
for (idx_t i = 0; i < ghost.shape(0); ++i) { |
| 68 |
ghost(i) = partition(i) != thisPart || remote_index(i) != i; |
| 69 |
} |
| 70 |
return ghost_field; |
| 71 |
} |
| 72 |
return functionspace.ghost(); |
| 73 |
} |
| 74 |
|
| 75 |
|
| 76 |
// Create UID field. |
| 77 |
Field getUidField(const FunctionSpace& functionspace) { |
| 78 |
// Note there is a difference between uid_field and global_index in the type: uidx_t vs gidx_t |
| 79 |
// Currently this aliases the same type though... |
| 80 |
|
| 81 |
if (not functionspace::PointCloud(functionspace)) { |
| 82 |
return functionspace.global_index(); |
| 83 |
} |
| 84 |
else { |
| 85 |
// PointCloud has no global_index()... TODO, put something in place there. |
| 86 |
|
| 87 |
// Make UID field. |
| 88 |
Field uid_field = Field("Unique ID", array::make_datatype<uidx_t>(), array::make_shape(functionspace.size())); |
| 89 |
|
| 90 |
// Make views. |
| 91 |
auto uid = array::make_view<uidx_t, 1>(uid_field); |
| 92 |
const auto lonlat = array::make_view<double, 2>(functionspace.lonlat()); |
| 93 |
|
| 94 |
// Set UIDs |
| 95 |
for (idx_t i = 0; i < uid_field.shape(0); ++i) { |
| 96 |
uid(i) = util::unique_lonlat(lonlat(i, LON), lonlat(i, LAT)); |
| 97 |
} |
| 98 |
return uid_field; |
| 99 |
} |
| 100 |
} |
| 101 |
|
| 102 |
// Resolve functionspace implementation type and get unique ID vector. |
| 103 |
std::vector<IdxUid> getUidVec(const FunctionSpace& functionspace) { |
| 104 |
// Get ghost and unique ID fields from functionspace. |
| 105 |
const auto ghost_field = getGhostField(functionspace); |
| 106 |
const auto uid = getUidField(functionspace); |
| 107 |
|
| 108 |
// Make views to fields. |
| 109 |
const auto ghost = array::make_view<int, 1>(ghost_field); |
| 110 |
const auto uidView = array::make_view<uidx_t, 1>(uid); |
| 111 |
|
| 112 |
auto uidVec = std::vector<IdxUid>{}; |
| 113 |
uidVec.reserve(functionspace.size()); |
| 114 |
|
| 115 |
// Get UIDs for non ghost elems. |
| 116 |
for (idx_t i = 0; i < functionspace.size(); ++i) { |
| 117 |
if (not ghost(i)) { |
| 118 |
uidVec.emplace_back(i, uidView(i)); |
| 119 |
} |
| 120 |
} |
| 121 |
|
| 122 |
// Sort by UID value. |
| 123 |
std::sort(uidVec.begin(), uidVec.end(), [](const IdxUid& a, const IdxUid& b) { return a.second < b.second; }); |
| 124 |
|
| 125 |
// Check UIDs are unique. |
| 126 |
if (ATLAS_BUILD_TYPE_DEBUG) { |
| 127 |
auto first_duplicate = std::adjacent_find( |
| 128 |
uidVec.begin(), uidVec.end(), [](const IdxUid& a, const IdxUid& b) { return a.second == b.second; }); |
| 129 |
ATLAS_ASSERT(uidVec.end() == first_duplicate, "Unique ID set has duplicate members"); |
| 130 |
} |
| 131 |
|
| 132 |
return uidVec; |
| 133 |
} |
| 134 |
|
| 135 |
// Copy UID index |
| 136 |
std::vector<idx_t> getUidIdx(const std::vector<IdxUid>& uidVec) { |
| 137 |
auto idxVec = std::vector<idx_t>{}; |
| 138 |
idxVec.reserve(uidVec.size()); |
| 139 |
std::transform(uidVec.begin(), uidVec.end(), std::back_inserter(idxVec), |
| 140 |
[](const IdxUid& uid) { return uid.first; }); |
| 141 |
return idxVec; |
| 142 |
} |
| 143 |
|
| 144 |
// Copy UID value |
| 145 |
std::vector<uidx_t> getUidVal(const std::vector<IdxUid>& uidVec) { |
| 146 |
auto valVec = std::vector<uidx_t>{}; |
| 147 |
valVec.reserve(uidVec.size()); |
| 148 |
std::transform(uidVec.begin(), uidVec.end(), std::back_inserter(valVec), |
| 149 |
[](const IdxUid& uid) { return uid.second; }); |
| 150 |
return valVec; |
| 151 |
} |
| 152 |
|
| 153 |
// Communicate UID values, return receive buffer and displacements. |
| 154 |
std::pair<std::vector<uidx_t>, std::vector<int>> communicateUid(const std::string& mpi_comm, const std::vector<uidx_t>& sendBuffer) { |
| 155 |
auto& comm = mpi::comm(mpi_comm); |
| 156 |
auto counts = std::vector<int>(comm.size()); |
| 157 |
comm.allGather(static_cast<int>(sendBuffer.size()), counts.begin(), counts.end()); |
| 158 |
|
| 159 |
auto disps = std::vector<int>{}; |
| 160 |
disps.reserve(comm.size() + 1); |
| 161 |
disps.push_back(0); |
| 162 |
std::partial_sum(counts.begin(), counts.end(), std::back_inserter(disps)); |
| 163 |
|
| 164 |
|
| 165 |
auto recvBuffer = std::vector<uidx_t>(static_cast<size_t>(disps.back())); |
| 166 |
|
| 167 |
comm.allGatherv(sendBuffer.begin(), sendBuffer.end(), recvBuffer.begin(), counts.data(), disps.data()); |
| 168 |
|
| 169 |
return std::make_pair(recvBuffer, disps); |
| 170 |
} |
| 171 |
|
| 172 |
// Need to overload "<" operator for comparison of asymmetric types in |
| 173 |
// std::set_intersection. |
| 174 |
bool operator<(const IdxUid& lhs, const uidx_t& rhs) { |
| 175 |
return lhs.second < rhs; |
| 176 |
} |
| 177 |
bool operator<(const uidx_t& lhs, const IdxUid& rhs) { |
| 178 |
return lhs < rhs.second; |
| 179 |
} |
| 180 |
|
| 181 |
// Find the intersection between local and global UIDs, then return local |
| 182 |
// indices of incections and PE dispacements in vector. |
| 183 |
std::pair<std::vector<idx_t>, std::vector<int>> getUidIntersection(const std::string& mpi_comm, |
| 184 |
const std::vector<IdxUid>& localUids, |
| 185 |
const std::vector<uidx_t>& globalUids, |
| 186 |
const std::vector<int>& globalDisps) { |
| 187 |
auto uidIntersection = std::vector<IdxUid>{}; |
| 188 |
uidIntersection.reserve(localUids.size()); |
| 189 |
|
| 190 |
auto& comm = mpi::comm(mpi_comm); |
| 191 |
auto mpi_size = comm.size(); |
| 192 |
|
| 193 |
auto disps = std::vector<int>{}; |
| 194 |
disps.reserve(mpi_size + 1); |
| 195 |
disps.push_back(0); |
| 196 |
|
| 197 |
// Loop over all PE and find UID intersection. |
| 198 |
for (size_t i = 0; i < mpi_size; ++i) { |
| 199 |
// Get displaced iterators. |
| 200 |
auto globalUidsBegin = globalUids.begin() + globalDisps[i]; |
| 201 |
auto globalUidsEnd = globalUids.begin() + globalDisps[i + 1]; |
| 202 |
|
| 203 |
// Get intersection. |
| 204 |
std::set_intersection(localUids.begin(), localUids.end(), globalUidsBegin, globalUidsEnd, |
| 205 |
std::back_inserter(uidIntersection)); |
| 206 |
|
| 207 |
// Set intersection displacement. |
| 208 |
disps.push_back(static_cast<int>(uidIntersection.size())); |
| 209 |
} |
| 210 |
|
| 211 |
// Check that the set of all intersections matches UIDs on local PE. |
| 212 |
if (ATLAS_BUILD_TYPE_DEBUG) { |
| 213 |
auto tempUids = uidIntersection; |
| 214 |
std::sort(tempUids.begin(), tempUids.end(), |
| 215 |
[](const IdxUid& a, const IdxUid& b) { return a.second < b.second; }); |
| 216 |
ATLAS_ASSERT(tempUids == localUids, "Set of all UID intersections does not match local UIDs."); |
| 217 |
} |
| 218 |
|
| 219 |
// Return local indices of intersection and displacements. |
| 220 |
return make_pair(getUidIdx(uidIntersection), disps); |
| 221 |
} |
| 222 |
|
| 223 |
|
| 224 |
// Iterate over a field, in the order of an index list, and apply a functor to |
| 225 |
// each element. |
| 226 |
|
| 227 |
// Recursive ForEach to visit all elements of field. |
| 228 |
template <int Rank, int Dim = 0> |
| 229 |
struct ForEach { |
| 230 |
template <typename Value, typename Functor, typename... Idxs> |
| 231 |
static void apply(const std::vector<idx_t>& idxList, array::ArrayView<Value, Rank>& fieldView, const Functor& f, |
| 232 |
Idxs... idxs) { |
| 233 |
// Iterate over dimension Dim of array. |
| 234 |
for (idx_t idx = 0; idx < fieldView.shape(Dim); ++idx) { |
| 235 |
ForEach<Rank, Dim + 1>::apply(idxList, fieldView, f, idxs..., idx); |
| 236 |
} |
| 237 |
} |
| 238 |
}; |
| 239 |
|
| 240 |
// Beginning of recursion when Dim == 0. |
| 241 |
template <int Rank> |
| 242 |
struct ForEach<Rank, 0> { |
| 243 |
template <typename Value, typename Functor, typename... Idxs> |
| 244 |
static void apply(const std::vector<idx_t>& idxList, array::ArrayView<Value, Rank>& fieldView, const Functor& f, |
| 245 |
Idxs... idxs) { |
| 246 |
// Iterate over dimension 0 of array in order defined by idxList. |
| 247 |
for (idx_t idx : idxList) { |
| 248 |
ForEach<Rank, 1>::apply(idxList, fieldView, f, idxs..., idx); |
| 249 |
} |
| 250 |
} |
| 251 |
}; |
| 252 |
|
| 253 |
// End of recursion when Dim == Rank. |
| 254 |
template <int Rank> |
| 255 |
struct ForEach<Rank, Rank> { |
| 256 |
template <typename Value, typename Functor, typename... Idxs> |
| 257 |
static void apply(const std::vector<idx_t>& idxList, array::ArrayView<Value, Rank>& fieldView, const Functor& f, |
| 258 |
Idxs... idxs) { |
| 259 |
// Apply functor. |
| 260 |
f(fieldView(idxs...)); |
| 261 |
} |
| 262 |
}; |
| 263 |
|
| 264 |
} // namespace |
| 265 |
|
| 266 |
void RedistributeGeneric::do_setup() { |
| 267 |
ATLAS_ASSERT( source().mpi_comm() == target().mpi_comm() ); |
| 268 |
|
| 269 |
mpi_comm_ = source().mpi_comm(); |
| 270 |
|
| 271 |
// get a unique ID (UID) for each owned member of functionspace. |
| 272 |
const auto sourceUidVec = getUidVec(source()); |
| 273 |
const auto targetUidVec = getUidVec(target()); |
| 274 |
|
| 275 |
// Communicate UID vectors to all PEs. |
| 276 |
auto sourceGlobalUids = std::vector<uidx_t>{}; |
| 277 |
auto sourceGlobalDisps = std::vector<int>{}; |
| 278 |
std::tie(sourceGlobalUids, sourceGlobalDisps) = communicateUid(mpi_comm_, getUidVal(sourceUidVec)); |
| 279 |
auto targetGlobalUids = std::vector<uidx_t>{}; |
| 280 |
auto targetGlobalDisps = std::vector<int>{}; |
| 281 |
std::tie(targetGlobalUids, targetGlobalDisps) = communicateUid(mpi_comm_, getUidVal(targetUidVec)); |
| 282 |
|
| 283 |
// Get intersection of local UIDs and Global UIDs. |
| 284 |
std::tie(sourceLocalIdx_, sourceDisps_) = getUidIntersection(mpi_comm_, sourceUidVec, targetGlobalUids, targetGlobalDisps); |
| 285 |
std::tie(targetLocalIdx_, targetDisps_) = getUidIntersection(mpi_comm_, targetUidVec, sourceGlobalUids, sourceGlobalDisps); |
| 286 |
} |
| 287 |
|
| 288 |
void RedistributeGeneric::execute(const Field& sourceField, Field& targetField) const { |
| 289 |
//Check functionspaces match. |
| 290 |
ATLAS_ASSERT(sourceField.functionspace().type() == source().type()); |
| 291 |
ATLAS_ASSERT(targetField.functionspace().type() == target().type()); |
| 292 |
|
| 293 |
// Check Field datatypes match. |
| 294 |
ATLAS_ASSERT(sourceField.datatype() == targetField.datatype()); |
| 295 |
|
| 296 |
// Check Field ranks match. |
| 297 |
ATLAS_ASSERT(sourceField.rank() == targetField.rank()); |
| 298 |
|
| 299 |
// Check number of levels and variables match. |
| 300 |
for (idx_t i = 1; i < sourceField.rank(); ++i) { |
| 301 |
ATLAS_ASSERT(sourceField.shape(i) == targetField.shape(i)); |
| 302 |
} |
| 303 |
|
| 304 |
// Perform redistribution. |
| 305 |
do_execute(sourceField, targetField); |
| 306 |
} |
| 307 |
|
| 308 |
void RedistributeGeneric::execute(const FieldSet& sourceFieldSet, FieldSet& targetFieldSet) const { |
| 309 |
// Check field set sizes match. |
| 310 |
ATLAS_ASSERT(sourceFieldSet.size() == targetFieldSet.size()); |
| 311 |
|
| 312 |
// Redistribute fields. |
| 313 |
for (idx_t i = 0; i < sourceFieldSet.size(); ++i) { |
| 314 |
execute(sourceFieldSet[i], targetFieldSet[i]); |
| 315 |
} |
| 316 |
} |
| 317 |
|
| 318 |
// Determine datatype. |
| 319 |
void RedistributeGeneric::do_execute(const Field& sourceField, Field& targetField) const { |
| 320 |
// Available datatypes defined in array/LocalView.cc |
| 321 |
switch (sourceField.datatype().kind()) { |
| 322 |
case array::DataType::KIND_REAL64: { |
| 323 |
return do_execute<double>(sourceField, targetField); |
| 324 |
} |
| 325 |
case array::DataType::KIND_REAL32: { |
| 326 |
return do_execute<float>(sourceField, targetField); |
| 327 |
} |
| 328 |
case array::DataType::KIND_INT64: { |
| 329 |
return do_execute<long>(sourceField, targetField); |
| 330 |
} |
| 331 |
case array::DataType::KIND_INT32: { |
| 332 |
return do_execute<int>(sourceField, targetField); |
| 333 |
} |
| 334 |
default: { |
| 335 |
ATLAS_THROW_EXCEPTION("No implementation for data type " + sourceField.datatype().str()); |
| 336 |
} |
| 337 |
} |
| 338 |
} |
| 339 |
|
| 340 |
// Determine rank. |
| 341 |
template <typename Value> |
| 342 |
void RedistributeGeneric::do_execute(const Field& sourceField, Field& targetField) const { |
| 343 |
// Available ranks defined in array/LocalView.cc |
| 344 |
switch (sourceField.rank()) { |
| 345 |
case 1: { |
| 346 |
return do_execute<Value, 1>(sourceField, targetField); |
| 347 |
} |
| 348 |
case 2: { |
| 349 |
return do_execute<Value, 2>(sourceField, targetField); |
| 350 |
} |
| 351 |
case 3: { |
| 352 |
return do_execute<Value, 3>(sourceField, targetField); |
| 353 |
} |
| 354 |
case 4: { |
| 355 |
return do_execute<Value, 4>(sourceField, targetField); |
| 356 |
} |
| 357 |
case 5: { |
| 358 |
return do_execute<Value, 5>(sourceField, targetField); |
| 359 |
} |
| 360 |
case 6: { |
| 361 |
return do_execute<Value, 6>(sourceField, targetField); |
| 362 |
} |
| 363 |
case 7: { |
| 364 |
return do_execute<Value, 7>(sourceField, targetField); |
| 365 |
} |
| 366 |
case 8: { |
| 367 |
return do_execute<Value, 8>(sourceField, targetField); |
| 368 |
} |
| 369 |
case 9: { |
| 370 |
return do_execute<Value, 9>(sourceField, targetField); |
| 371 |
} |
| 372 |
default: { |
| 373 |
ATLAS_THROW_EXCEPTION("No implementation for rank " + std::to_string(sourceField.rank())); |
| 374 |
} |
| 375 |
} |
| 376 |
} |
| 377 |
|
| 378 |
// Perform redistribution. |
| 379 |
template <typename Value, int Rank> |
| 380 |
void RedistributeGeneric::do_execute(const Field& sourceField, Field& targetField) const { |
| 381 |
// Get array views. |
| 382 |
auto sourceView = array::make_view<Value, Rank>(sourceField); |
| 383 |
auto targetView = array::make_view<Value, Rank>(targetField); |
| 384 |
|
| 385 |
const auto& comm = mpi::comm(mpi_comm_); |
| 386 |
auto mpi_size = comm.size(); |
| 387 |
|
| 388 |
// Get number of elems per column. |
| 389 |
int elemsPerCol = 1; |
| 390 |
for (int i = 1; i < Rank; ++i) { |
| 391 |
elemsPerCol *= sourceView.shape(i); |
| 392 |
} |
| 393 |
|
| 394 |
// Set send displacement and counts vectors. |
| 395 |
auto sendDisps = std::vector<int>{}; |
| 396 |
sendDisps.reserve(mpi_size + 1); |
| 397 |
auto sendCounts = std::vector<int>{}; |
| 398 |
sendCounts.reserve(mpi_size); |
| 399 |
std::transform(sourceDisps_.begin(), sourceDisps_.end(), std::back_inserter(sendDisps), |
| 400 |
[&](const int& disp) { return disp * elemsPerCol; }); |
| 401 |
std::adjacent_difference(sendDisps.begin() + 1, sendDisps.end(), std::back_inserter(sendCounts)); |
| 402 |
|
| 403 |
// Set recv displacement and counts vectors. |
| 404 |
auto recvDisps = std::vector<int>{}; |
| 405 |
recvDisps.reserve(mpi_size + 1); |
| 406 |
auto recvCounts = std::vector<int>{}; |
| 407 |
recvCounts.reserve(mpi_size); |
| 408 |
std::transform(targetDisps_.begin(), targetDisps_.end(), std::back_inserter(recvDisps), |
| 409 |
[&](const int& disp) { return disp * elemsPerCol; }); |
| 410 |
std::adjacent_difference(recvDisps.begin() + 1, recvDisps.end(), std::back_inserter(recvCounts)); |
| 411 |
|
| 412 |
// Allocate send and recv buffers. |
| 413 |
auto sendBuffer = std::vector<Value>(static_cast<size_t>(sendDisps.back())); |
| 414 |
auto recvBuffer = std::vector<Value>(static_cast<size_t>(recvDisps.back())); |
| 415 |
auto sendBufferIt = sendBuffer.begin(); |
| 416 |
auto recvBufferIt = recvBuffer.cbegin(); |
| 417 |
|
| 418 |
// Copy sourceField to sendBuffer. |
| 419 |
ForEach<Rank>::apply(sourceLocalIdx_, sourceView, [&](const Value& elem) { *sendBufferIt++ = elem; }); |
| 420 |
|
| 421 |
// Perform MPI communication. |
| 422 |
comm.allToAllv(sendBuffer.data(), sendCounts.data(), sendDisps.data(), recvBuffer.data(), recvCounts.data(), |
| 423 |
recvDisps.data()); |
| 424 |
|
| 425 |
// Copy recvBuffer to targetField. |
| 426 |
ForEach<Rank>::apply(targetLocalIdx_, targetView, [&](Value& elem) { elem = *recvBufferIt++; }); |
| 427 |
} |
| 428 |
|
| 429 |
namespace { |
| 430 |
static RedistributionImplBuilder<RedistributeGeneric> register_builder(RedistributeGeneric::static_type()); |
| 431 |
} |
| 432 |
|
| 433 |
} // namespace detail |
| 434 |
} // namespace redistribution |
| 435 |
} // namespace atlas |