Debian Codegraph @

src/atlas/grid/detail/partitioner/MatchingMeshPartitionerLonLatPolygon.cc

raw
1 /*
2 * (C) Copyright 2013 ECMWF.
3 *
4 * This software is licensed under the terms of the Apache Licence Version 2.0
5 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
6 * In applying this licence, ECMWF does not waive the privileges and immunities
7 * granted to it by virtue of its status as an intergovernmental organisation
8 * nor does it submit to any jurisdiction.
9 */
10
11 #include "atlas/grid/detail/partitioner/MatchingMeshPartitionerLonLatPolygon.h"
12
13 #include <iomanip>
14 #include <sstream>
15 #include <vector>
16
17 #include "eckit/config/Resource.h"
18 #include "eckit/log/ProgressTimer.h"
19
20 #include "atlas/grid/Grid.h"
21 #include "atlas/grid/Iterator.h"
22 #include "atlas/mesh/Nodes.h"
23 #include "atlas/parallel/mpi/mpi.h"
24 #include "atlas/parallel/omp/fill.h"
25 #include "atlas/parallel/omp/omp.h"
26 #include "atlas/runtime/Exception.h"
27 #include "atlas/runtime/Log.h"
28 #include "atlas/util/CoordinateEnums.h"
29 #include "atlas/util/PolygonXY.h"
30
31
32 #include "atlas/util/KDTree.h"
33
34 namespace atlas {
35 namespace grid {
36 namespace detail {
37 namespace partitioner {
38
39 namespace {
40 PartitionerBuilder<MatchingMeshPartitionerLonLatPolygon> __builder("lonlat-polygon");
41 }
42
43 MatchingMeshPartitionerLonLatPolygon::MatchingMeshPartitionerLonLatPolygon(const Mesh& mesh, const eckit::Parametrisation& config):
44 MatchingMeshPartitioner(mesh, config) {
45 config.get("fallback_nearest", fallback_nearest_);
46 }
47
48
49 void MatchingMeshPartitionerLonLatPolygon::partition(const Grid& grid, int partitioning[]) const {
50 const auto& comm = mpi::comm(prePartitionedMesh_.mpi_comm());
51 const int mpi_rank = int(comm.rank());
52
53 ATLAS_TRACE("MatchingMeshPartitionerLonLatPolygon::partition");
54
55 ATLAS_ASSERT(grid.domain().global());
56
57 Log::debug() << "MatchingMeshPartitionerLonLatPolygon::partition" << std::endl;
58
59 const util::PolygonXY poly{prePartitionedMesh_.polygon(0)};
60
61 double west = poly.coordinatesMin().x();
62 double east = poly.coordinatesMax().x();
63 ATLAS_TRACE_MPI(ALLREDUCE) {
64 comm.allReduceInPlace(west, eckit::mpi::Operation::MIN);
65 comm.allReduceInPlace(east, eckit::mpi::Operation::MAX);
66 }
67
68 Projection projection = prePartitionedMesh_.projection();
69 omp::fill(partitioning, partitioning + grid.size(), -1);
70
71 auto compute = [&](double west) {
72 #if !defined(__NVCOMPILER)
73 atlas_omp_parallel
74 #elif (__NVCOMPILER_MAJOR__ >= 21) && (__NVCOMPILER_MINOR__ > 9 )
75 // Internal compiler error with nvhpc 21.9:
76 //
77 // NVC++-S-0000-Internal compiler error. BAD sptr in var_refsym 0 (MatchingMeshPartitionerLonLatPolygon.cc: 64) (=following line)
78 // NVC++/x86-64 Linux 21.9-0: compilation completed with severe errors
79 atlas_omp_parallel
80 #endif
81 {
82 const idx_t num_threads = atlas_omp_get_num_threads();
83 const idx_t thread_num = atlas_omp_get_thread_num();
84 const idx_t begin = static_cast<idx_t>(thread_num * size_t(grid.size()) / num_threads);
85 const idx_t end =
86 static_cast<idx_t>((thread_num + 1) * size_t(grid.size()) / num_threads);
87 size_t i = begin;
88 auto it = grid.lonlat().begin() + i;
89 for(; i < end; ++i, ++it) {
90 PointLonLat P = *it;
91 if (partitioning[i] < 0) {
92 projection.lonlat2xy(P);
93 P.normalise(west);
94 partitioning[i] = poly.contains(P) ? mpi_rank : -1;
95 }
96 }
97 }
98 // Synchronize partitioning
99 ATLAS_TRACE_MPI(ALLREDUCE) {
100 comm.allReduceInPlace(partitioning, grid.size(), eckit::mpi::Operation::MAX);
101 }
102
103 std::vector<int> thread_min(atlas_omp_get_max_threads(),std::numeric_limits<int>::max());
104 #if !defined(__NVCOMPILER)
105 atlas_omp_parallel
106 #elif (__NVCOMPILER_MAJOR__ >= 21) && (__NVCOMPILER_MINOR__ > 9 )
107 // Internal compiler error with nvhpc 21.9:
108 //
109 // NVC++-S-0000-Internal compiler error. BAD sptr in var_refsym 0 (MatchingMeshPartitionerLonLatPolygon.cc: 64) (=following line)
110 // NVC++/x86-64 Linux 21.9-0: compilation completed with severe errors
111 atlas_omp_parallel
112 #endif
113 {
114 const idx_t num_threads = atlas_omp_get_num_threads();
115 const idx_t thread_num = atlas_omp_get_thread_num();
116 const idx_t begin = static_cast<idx_t>(thread_num * size_t(grid.size()) / num_threads);
117 const idx_t end =
118 static_cast<idx_t>((thread_num + 1) * size_t(grid.size()) / num_threads);
119 thread_min[thread_num] = *std::min_element(partitioning+begin,partitioning+end);
120 }
121 return *std::min_element(thread_min.begin(), thread_min.end());
122 };
123
124 int min = compute(east - 360.);
125 constexpr double eps = 1.e-10;
126 bool second_try = [&]() {
127 if (min < 0 && east - west > 360. + eps) {
128 min = compute(west - eps);
129 return true;
130 }
131 return false;
132 }();
133
134 if (min < 0) {
135 size_t i = 0;
136 size_t max_failures = grid.size();
137 std::vector<size_t> failed_index;
138 std::vector<PointLonLat> failed_lonlat;
139 failed_index.reserve(max_failures);
140 failed_lonlat.reserve(max_failures);
141 for (PointLonLat P : grid.lonlat()) {
142 if (partitioning[i] < 0) {
143 failed_index.emplace_back(i);
144 failed_lonlat.emplace_back(P);
145 }
146 ++i;
147 }
148 size_t nb_failures = failed_index.size();
149
150 if (fallback_nearest_) {
151 util::IndexKDTree3D kdtree;
152 kdtree.reserve(grid.size());
153 size_t j=0;
154 for (auto& p: grid.lonlat()) {
155 if (partitioning[j] >= 0) {
156 kdtree.insert(p,partitioning[j]);
157 }
158 ++j;
159 }
160 kdtree.build();
161 for (size_t n = 0; n < nb_failures; ++n) {
162 auto closest = kdtree.closestPoint(failed_lonlat[n]);
163 partitioning[failed_index[n]] = closest.payload();
164 }
165 }
166 else {
167 std::stringstream err;
168 err.precision(20);
169 err << "Could not find partition of " << nb_failures
170 << " target grid points (source mesh does not contain all target grid points)\n"
171 << "Tried first normalizing coordinates with west=" << east - 360. << "\n";
172 if (second_try) {
173 err << "Tried second time normalizing coordinates with west=" << west - eps << "\n";
174 }
175
176 err << "Failed target grid points with global index:\n";
177 for (size_t n = 0; n < nb_failures; ++n) {
178 err << " - " << std::setw(10) << std::left << failed_index[n] + 1 << " {lon,lat} : " << failed_lonlat[n]
179 << "\n";
180 }
181 // prePartitionedMesh_.polygon(0).outputPythonScript("partitions.py");
182 throw_Exception(err.str(), Here());
183 }
184 }
185 }
186
187 } // namespace partitioner
188 } // namespace detail
189 } // namespace grid
190 } // namespace atlas