| 1 |
/* |
| 2 |
* (C) Crown Copyright 2022 Met Office |
| 3 |
* |
| 4 |
* This software is licensed under the terms of the Apache Licence Version 2.0 |
| 5 |
* which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. |
| 6 |
*/ |
| 7 |
|
| 8 |
#include "atlas/grid/detail/partitioner/MatchingMeshPartitionerCubedSphere.h" |
| 9 |
|
| 10 |
#include <algorithm> |
| 11 |
#include <iterator> |
| 12 |
#include <vector> |
| 13 |
|
| 14 |
#include "atlas/grid/CubedSphereGrid.h" |
| 15 |
#include "atlas/grid/Iterator.h" |
| 16 |
#include "atlas/interpolation/method/cubedsphere/CellFinder.h" |
| 17 |
#include "atlas/parallel/mpi/mpi.h" |
| 18 |
#include "atlas/parallel/omp/omp.h" |
| 19 |
#include "atlas/util/Point.h" |
| 20 |
|
| 21 |
namespace atlas { |
| 22 |
namespace grid { |
| 23 |
namespace detail { |
| 24 |
namespace partitioner { |
| 25 |
|
| 26 |
void MatchingMeshPartitionerCubedSphere::partition(const Grid& grid, int partitioning[]) const { |
| 27 |
const auto& comm = mpi::comm(prePartitionedMesh_.mpi_comm()); |
| 28 |
const int mpi_rank = int(comm.rank()); |
| 29 |
|
| 30 |
// Make cell finder from owned mesh cells. |
| 31 |
const auto finder = interpolation::method::cubedsphere::CellFinder(prePartitionedMesh_); |
| 32 |
|
| 33 |
// Numeric tolerance should scale with N. |
| 34 |
const auto N = CubedSphereGrid(prePartitionedMesh_.grid()).N(); |
| 35 |
const auto epsilon = 2. * std::numeric_limits<double>::epsilon() * N; |
| 36 |
const auto edgeEpsilon = epsilon; |
| 37 |
const size_t listSize = 8; |
| 38 |
|
| 39 |
|
| 40 |
const auto setPartitioning = [&](const auto& lonLatIt) { |
| 41 |
atlas_omp_parallel_for(gidx_t i = 0; i < grid.size(); ++i) { |
| 42 |
const auto lonLat = *(lonLatIt + i); |
| 43 |
// This is probably more expensive than it needs to be, as it performs |
| 44 |
// a dry run of the cubedsphere interpolation method. |
| 45 |
partitioning[i] = finder.getCell(lonLat, listSize, edgeEpsilon, epsilon).isect ? mpi_rank : -1; |
| 46 |
} |
| 47 |
}; |
| 48 |
|
| 49 |
// CubedSphereIterator::operator+=() is not implemented properly. |
| 50 |
if (CubedSphereGrid(grid)) { |
| 51 |
auto lonLats = std::vector<PointLonLat>{}; |
| 52 |
lonLats.reserve(grid.size()); |
| 53 |
std::copy(grid.lonlat().begin(), grid.lonlat().end(), std::back_inserter(lonLats)); |
| 54 |
setPartitioning(lonLats.begin()); |
| 55 |
} else { |
| 56 |
setPartitioning(grid.lonlat().begin()); |
| 57 |
} |
| 58 |
|
| 59 |
// AllReduce to get full partitioning array. |
| 60 |
comm.allReduceInPlace(partitioning, grid.size(), eckit::mpi::Operation::MAX); |
| 61 |
const auto misses = std::count_if(partitioning, partitioning + grid.size(), [](int elem) { return elem < 0; }); |
| 62 |
if (misses > 0) { |
| 63 |
throw_Exception( |
| 64 |
"Could not find partition for target node (source " |
| 65 |
"mesh does not contain all target grid points)\n" + |
| 66 |
std::to_string(misses) + " misses.", |
| 67 |
Here()); |
| 68 |
} |
| 69 |
} |
| 70 |
|
| 71 |
} // namespace partitioner |
| 72 |
} // namespace detail |
| 73 |
} // namespace grid |
| 74 |
} // namespace atlas |