| 1 |
/* |
| 2 |
* (C) Copyright 2013 ECMWF. |
| 3 |
* |
| 4 |
* This software is licensed under the terms of the Apache Licence Version 2.0 |
| 5 |
* which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. |
| 6 |
* In applying this licence, ECMWF does not waive the privileges and immunities |
| 7 |
* granted to it by virtue of its status as an intergovernmental organisation |
| 8 |
* nor does it submit to any jurisdiction. |
| 9 |
*/ |
| 10 |
|
| 11 |
#include "atlas/grid/detail/partitioner/MatchingMeshPartitionerLonLatPolygon.h" |
| 12 |
|
| 13 |
#include <iomanip> |
| 14 |
#include <sstream> |
| 15 |
#include <vector> |
| 16 |
|
| 17 |
#include "eckit/config/Resource.h" |
| 18 |
#include "eckit/log/ProgressTimer.h" |
| 19 |
|
| 20 |
#include "atlas/grid/Grid.h" |
| 21 |
#include "atlas/grid/Iterator.h" |
| 22 |
#include "atlas/mesh/Nodes.h" |
| 23 |
#include "atlas/parallel/mpi/mpi.h" |
| 24 |
#include "atlas/parallel/omp/fill.h" |
| 25 |
#include "atlas/parallel/omp/omp.h" |
| 26 |
#include "atlas/runtime/Exception.h" |
| 27 |
#include "atlas/runtime/Log.h" |
| 28 |
#include "atlas/util/CoordinateEnums.h" |
| 29 |
#include "atlas/util/PolygonXY.h" |
| 30 |
|
| 31 |
|
| 32 |
#include "atlas/util/KDTree.h" |
| 33 |
|
| 34 |
namespace atlas { |
| 35 |
namespace grid { |
| 36 |
namespace detail { |
| 37 |
namespace partitioner { |
| 38 |
|
| 39 |
namespace { |
| 40 |
PartitionerBuilder<MatchingMeshPartitionerLonLatPolygon> __builder("lonlat-polygon"); |
| 41 |
} |
| 42 |
|
| 43 |
MatchingMeshPartitionerLonLatPolygon::MatchingMeshPartitionerLonLatPolygon(const Mesh& mesh, const eckit::Parametrisation& config): |
| 44 |
MatchingMeshPartitioner(mesh, config) { |
| 45 |
config.get("fallback_nearest", fallback_nearest_); |
| 46 |
} |
| 47 |
|
| 48 |
|
| 49 |
void MatchingMeshPartitionerLonLatPolygon::partition(const Grid& grid, int partitioning[]) const { |
| 50 |
const auto& comm = mpi::comm(prePartitionedMesh_.mpi_comm()); |
| 51 |
const int mpi_rank = int(comm.rank()); |
| 52 |
|
| 53 |
ATLAS_TRACE("MatchingMeshPartitionerLonLatPolygon::partition"); |
| 54 |
|
| 55 |
ATLAS_ASSERT(grid.domain().global()); |
| 56 |
|
| 57 |
Log::debug() << "MatchingMeshPartitionerLonLatPolygon::partition" << std::endl; |
| 58 |
|
| 59 |
const util::PolygonXY poly{prePartitionedMesh_.polygon(0)}; |
| 60 |
|
| 61 |
double west = poly.coordinatesMin().x(); |
| 62 |
double east = poly.coordinatesMax().x(); |
| 63 |
ATLAS_TRACE_MPI(ALLREDUCE) { |
| 64 |
comm.allReduceInPlace(west, eckit::mpi::Operation::MIN); |
| 65 |
comm.allReduceInPlace(east, eckit::mpi::Operation::MAX); |
| 66 |
} |
| 67 |
|
| 68 |
Projection projection = prePartitionedMesh_.projection(); |
| 69 |
omp::fill(partitioning, partitioning + grid.size(), -1); |
| 70 |
|
| 71 |
auto compute = [&](double west) { |
| 72 |
#if !defined(__NVCOMPILER) |
| 73 |
atlas_omp_parallel |
| 74 |
#elif (__NVCOMPILER_MAJOR__ >= 21) && (__NVCOMPILER_MINOR__ > 9 ) |
| 75 |
// Internal compiler error with nvhpc 21.9: |
| 76 |
// |
| 77 |
// NVC++-S-0000-Internal compiler error. BAD sptr in var_refsym 0 (MatchingMeshPartitionerLonLatPolygon.cc: 64) (=following line) |
| 78 |
// NVC++/x86-64 Linux 21.9-0: compilation completed with severe errors |
| 79 |
atlas_omp_parallel |
| 80 |
#endif |
| 81 |
{ |
| 82 |
const idx_t num_threads = atlas_omp_get_num_threads(); |
| 83 |
const idx_t thread_num = atlas_omp_get_thread_num(); |
| 84 |
const idx_t begin = static_cast<idx_t>(thread_num * size_t(grid.size()) / num_threads); |
| 85 |
const idx_t end = |
| 86 |
static_cast<idx_t>((thread_num + 1) * size_t(grid.size()) / num_threads); |
| 87 |
size_t i = begin; |
| 88 |
auto it = grid.lonlat().begin() + i; |
| 89 |
for(; i < end; ++i, ++it) { |
| 90 |
PointLonLat P = *it; |
| 91 |
if (partitioning[i] < 0) { |
| 92 |
projection.lonlat2xy(P); |
| 93 |
P.normalise(west); |
| 94 |
partitioning[i] = poly.contains(P) ? mpi_rank : -1; |
| 95 |
} |
| 96 |
} |
| 97 |
} |
| 98 |
// Synchronize partitioning |
| 99 |
ATLAS_TRACE_MPI(ALLREDUCE) { |
| 100 |
comm.allReduceInPlace(partitioning, grid.size(), eckit::mpi::Operation::MAX); |
| 101 |
} |
| 102 |
|
| 103 |
std::vector<int> thread_min(atlas_omp_get_max_threads(),std::numeric_limits<int>::max()); |
| 104 |
#if !defined(__NVCOMPILER) |
| 105 |
atlas_omp_parallel |
| 106 |
#elif (__NVCOMPILER_MAJOR__ >= 21) && (__NVCOMPILER_MINOR__ > 9 ) |
| 107 |
// Internal compiler error with nvhpc 21.9: |
| 108 |
// |
| 109 |
// NVC++-S-0000-Internal compiler error. BAD sptr in var_refsym 0 (MatchingMeshPartitionerLonLatPolygon.cc: 64) (=following line) |
| 110 |
// NVC++/x86-64 Linux 21.9-0: compilation completed with severe errors |
| 111 |
atlas_omp_parallel |
| 112 |
#endif |
| 113 |
{ |
| 114 |
const idx_t num_threads = atlas_omp_get_num_threads(); |
| 115 |
const idx_t thread_num = atlas_omp_get_thread_num(); |
| 116 |
const idx_t begin = static_cast<idx_t>(thread_num * size_t(grid.size()) / num_threads); |
| 117 |
const idx_t end = |
| 118 |
static_cast<idx_t>((thread_num + 1) * size_t(grid.size()) / num_threads); |
| 119 |
thread_min[thread_num] = *std::min_element(partitioning+begin,partitioning+end); |
| 120 |
} |
| 121 |
return *std::min_element(thread_min.begin(), thread_min.end()); |
| 122 |
}; |
| 123 |
|
| 124 |
int min = compute(east - 360.); |
| 125 |
constexpr double eps = 1.e-10; |
| 126 |
bool second_try = [&]() { |
| 127 |
if (min < 0 && east - west > 360. + eps) { |
| 128 |
min = compute(west - eps); |
| 129 |
return true; |
| 130 |
} |
| 131 |
return false; |
| 132 |
}(); |
| 133 |
|
| 134 |
if (min < 0) { |
| 135 |
size_t i = 0; |
| 136 |
size_t max_failures = grid.size(); |
| 137 |
std::vector<size_t> failed_index; |
| 138 |
std::vector<PointLonLat> failed_lonlat; |
| 139 |
failed_index.reserve(max_failures); |
| 140 |
failed_lonlat.reserve(max_failures); |
| 141 |
for (PointLonLat P : grid.lonlat()) { |
| 142 |
if (partitioning[i] < 0) { |
| 143 |
failed_index.emplace_back(i); |
| 144 |
failed_lonlat.emplace_back(P); |
| 145 |
} |
| 146 |
++i; |
| 147 |
} |
| 148 |
size_t nb_failures = failed_index.size(); |
| 149 |
|
| 150 |
if (fallback_nearest_) { |
| 151 |
util::IndexKDTree3D kdtree; |
| 152 |
kdtree.reserve(grid.size()); |
| 153 |
size_t j=0; |
| 154 |
for (auto& p: grid.lonlat()) { |
| 155 |
if (partitioning[j] >= 0) { |
| 156 |
kdtree.insert(p,partitioning[j]); |
| 157 |
} |
| 158 |
++j; |
| 159 |
} |
| 160 |
kdtree.build(); |
| 161 |
for (size_t n = 0; n < nb_failures; ++n) { |
| 162 |
auto closest = kdtree.closestPoint(failed_lonlat[n]); |
| 163 |
partitioning[failed_index[n]] = closest.payload(); |
| 164 |
} |
| 165 |
} |
| 166 |
else { |
| 167 |
std::stringstream err; |
| 168 |
err.precision(20); |
| 169 |
err << "Could not find partition of " << nb_failures |
| 170 |
<< " target grid points (source mesh does not contain all target grid points)\n" |
| 171 |
<< "Tried first normalizing coordinates with west=" << east - 360. << "\n"; |
| 172 |
if (second_try) { |
| 173 |
err << "Tried second time normalizing coordinates with west=" << west - eps << "\n"; |
| 174 |
} |
| 175 |
|
| 176 |
err << "Failed target grid points with global index:\n"; |
| 177 |
for (size_t n = 0; n < nb_failures; ++n) { |
| 178 |
err << " - " << std::setw(10) << std::left << failed_index[n] + 1 << " {lon,lat} : " << failed_lonlat[n] |
| 179 |
<< "\n"; |
| 180 |
} |
| 181 |
// prePartitionedMesh_.polygon(0).outputPythonScript("partitions.py"); |
| 182 |
throw_Exception(err.str(), Here()); |
| 183 |
} |
| 184 |
} |
| 185 |
} |
| 186 |
|
| 187 |
} // namespace partitioner |
| 188 |
} // namespace detail |
| 189 |
} // namespace grid |
| 190 |
} // namespace atlas |