Debian Codegraph @

src/atlas/grid/detail/partitioner/CubedSpherePartitioner.cc

raw
1 /*
2 * (C) Crown Copyright 2021 Met Office
3 *
4 * This software is licensed under the terms of the Apache Licence Version 2.0
5 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
6 * In applying this licence, ECMWF does not waive the privileges and immunities
7 * granted to it by virtue of its status as an intergovernmental organisation
8 * nor does it submit to any jurisdiction.
9 */
10
11
12 #include "CubedSpherePartitioner.h"
13
14 #include <algorithm>
15 #include <cmath>
16 #include <ctime>
17 #include <functional>
18 #include <iostream>
19 #include <vector>
20
21 #include "atlas/grid/CubedSphereGrid.h"
22 #include "atlas/runtime/Exception.h"
23 #include "atlas/runtime/Log.h"
24
25 namespace atlas {
26 namespace grid {
27 namespace detail {
28 namespace partitioner {
29
30 namespace {
31
32 bool isNearInt(double value) {
33 const double diff = value - std::floor(value);
34 return (diff <= std::numeric_limits<double>::epsilon() || diff >= (1.0 - std::numeric_limits<double>::epsilon()));
35 }
36
37 bool isConfigSufficient(const eckit::Parametrisation& config) {
38 return !(config.has("starting rank on tile")) || !(config.has("final rank on tile")) || !(config.has("nprocx")) ||
39 !(config.has("nprocy"));
40 }
41
42 std::vector<std::vector<atlas::idx_t>> createOffset(const std::array<std::size_t, 6>& ngpt,
43 const std::array<idx_t, 6> nproc1D,
44 const std::array<idx_t, 6> maxDim1D,
45 const std::array<idx_t, 6> maxDim2D) {
46 std::vector<std::vector<atlas::idx_t>> offset;
47
48 idx_t totalNproc1D = std::accumulate(std::begin(nproc1D), std::end(nproc1D), 0);
49
50 // ngpbt - number of grid points per band and tile.
51 std::vector<std::size_t> ngpbt(static_cast<std::size_t>(totalNproc1D), 0);
52
53 idx_t j(0);
54 for (std::size_t t = 0; t < 6; ++t) {
55 idx_t jt = static_cast<idx_t>(j);
56 std::size_t nproc = static_cast<std::size_t>(nproc1D[t]);
57 std::vector<idx_t> offsetPerTile(nproc + 1, 0);
58
59 for (std::size_t proc1D = 0; proc1D < nproc; ++proc1D, ++j) {
60 ngpbt[static_cast<size_t>(j)] =
61 proc1D < nproc - 1
62 ? ngpt[t] / nproc
63 : ngpt[t] - static_cast<std::size_t>(std::accumulate(ngpbt.begin() + jt, ngpbt.begin() + j, 0));
64
65 offsetPerTile[proc1D] =
66 proc1D == 0 ? 0 : std::accumulate(ngpbt.begin() + jt, ngpbt.begin() + j, 0) / maxDim2D[t];
67 }
68 offsetPerTile[nproc] = maxDim1D[t];
69 offset.push_back(offsetPerTile);
70 }
71
72 return offset;
73 }
74
75 } // namespace
76
77 CubedSpherePartitioner::CubedSpherePartitioner(): Partitioner() {}
78
79 CubedSpherePartitioner::CubedSpherePartitioner(int N): Partitioner(N, util::NoConfig()), regular_{true} {}
80
81 CubedSpherePartitioner::CubedSpherePartitioner(const eckit::Parametrisation& config):
82 Partitioner(config), regular_{isConfigSufficient(config)} {
83 if (config.has("starting rank on tile") && config.has("final rank on tile") && config.has("nprocx") &&
84 config.has("nprocy")) {
85 config.get("starting rank on tile", globalProcStartPE_);
86 config.get("final rank on tile", globalProcEndPE_);
87 config.get("nprocx", nprocx_);
88 config.get("nprocy", nprocy_);
89 }
90 }
91
92 CubedSpherePartitioner::CubedSpherePartitioner(int N, const eckit::Parametrisation& config):
93 Partitioner(N, config), regular_{isConfigSufficient(config)} {
94 if (config.has("starting rank on tile") && config.has("final rank on tile") && config.has("nprocx") &&
95 config.has("nprocy")) {
96 config.get("starting rank on tile", globalProcStartPE_);
97 config.get("final rank on tile", globalProcEndPE_);
98 config.get("nprocx", nprocx_);
99 config.get("nprocy", nprocy_);
100 }
101 }
102
103 CubedSpherePartitioner::CubedSpherePartitioner(const int N, const std::vector<int>& globalProcStartPE,
104 const std::vector<int>& globalProcEndPE, const std::vector<int>& nprocx,
105 const std::vector<int>& nprocy):
106 Partitioner(N, util::NoConfig()),
107 globalProcStartPE_(globalProcStartPE.begin(), globalProcStartPE.end()),
108 globalProcEndPE_(globalProcEndPE.begin(), globalProcEndPE.end()),
109 nprocx_{nprocx.begin(), nprocx.end()},
110 nprocy_{nprocy.begin(), nprocy.end()},
111 regular_{false} {}
112
113
114 CubedSpherePartitioner::CubedSphere CubedSpherePartitioner::cubedsphere(const Grid& grid) const {
115 // grid dimensions
116 const CubedSphereGrid cg(grid);
117 if (!cg) {
118 throw_Exception("CubedSphere Partitioner only works for cubed sphere grids.", Here());
119 }
120
121 CubedSphere cb;
122
123 for (std::size_t t = 0; t < 6; ++t) {
124 cb.nx[t] = cg.N();
125 cb.ny[t] = cg.N();
126 }
127
128 atlas::idx_t nparts = nb_partitions();
129
130 if (regular_) {
131 // share PEs around tiles
132 // minRanksPerTile
133
134 idx_t ranksPerTile = nparts / 6;
135
136 idx_t reminder = nparts - 6 * ranksPerTile;
137
138 for (std::size_t t = 0; t < 6; ++t) {
139 cb.nproc[t] = ranksPerTile;
140 }
141
142 // round-robin;
143 std::size_t t{0};
144 while (reminder > 0) {
145 if (t == 6) {
146 t = 0;
147 }
148 cb.nproc[t] += 1;
149 t += 1;
150 reminder -= 1;
151 }
152
153 // now need to specify nprocx and nprocy.
154 // nproc is 0 for the tile we use default nprocx and nprocy = 1
155
156 // if we can square-root nproc and get an integer
157 // we use that for nprocx and nprocy
158 // otherwise we split just in nprocx and keep nprocy =1;
159
160 for (std::size_t t = 0; t < 6; ++t) {
161 if (cb.nproc[t] > 0) {
162 double sq = std::sqrt(static_cast<double>(cb.nproc[t]));
163 if (isNearInt(sq)) {
164 cb.nprocx[t] = static_cast<idx_t>(std::round(sq));
165 cb.nprocy[t] = static_cast<idx_t>(std::round(sq));
166 }
167 else {
168 cb.nprocx[t] = 1;
169 cb.nprocy[t] = cb.nproc[t];
170 }
171 }
172 }
173 cb.globalProcStartPE[0] = 0;
174 cb.globalProcEndPE[0] = cb.nproc[0] - 1;
175
176 for (size_t t = 1; t < 6; ++t) {
177 if (cb.nproc[t] == 0) {
178 cb.globalProcStartPE[t] = cb.globalProcEndPE[t - 1];
179 cb.globalProcEndPE[t] = cb.globalProcEndPE[t - 1];
180 }
181 else {
182 cb.globalProcStartPE[t] = cb.globalProcEndPE[t - 1] + 1;
183 cb.globalProcEndPE[t] = cb.globalProcStartPE[t] + cb.nproc[t] - 1;
184 }
185 }
186 }
187 else {
188 for (size_t t = 0; t < 6; ++t) {
189 cb.globalProcStartPE[t] = globalProcStartPE_[t];
190 cb.globalProcEndPE[t] = globalProcEndPE_[t];
191 cb.nprocx[t] = nprocx_[t];
192 cb.nprocy[t] = nprocy_[t];
193 }
194 cb.nproc[0] = globalProcEndPE_[0] + 1;
195 for (std::size_t t = 1; t < 6; ++t) {
196 cb.nproc[t] = cb.globalProcStartPE[t] == cb.globalProcEndPE[t - 1]
197 ? cb.nproc[t] = 0
198 : cb.globalProcEndPE[t] - cb.globalProcStartPE[t] + 1;
199 }
200 }
201 return cb;
202 }
203
204 void CubedSpherePartitioner::partition(CubedSphere& cb, const int nb_cells, const CellInt cells[], int part[]) const {
205 // ngpt number of grid points per tile (this is for cell centers where the number of cells are the same
206 std::size_t tileCells = static_cast<std::size_t>(nb_cells / 6);
207 std::array<std::size_t, 6> ngpt{tileCells, tileCells, tileCells, tileCells, tileCells, tileCells};
208
209 // xoffset - the xoffset per band per tile (including cb.nx[t])
210 //std::vector<std::vector<atlas::idx_t>> xoffset(createOffset(ngpt, cb.nprocx, cb.nx, cb.ny));
211
212 cb.xoffset = createOffset(ngpt, cb.nprocx, cb.nx, cb.ny);
213
214 // yoffset - the yoffset per band per tile (including cb.ny[t])
215 cb.yoffset = createOffset(ngpt, cb.nprocy, cb.ny, cb.nx);
216
217 int p; // partition index
218 // loop over all data tile by tile
219 for (int n = 0; n < nb_cells; ++n) {
220 std::size_t t = static_cast<std::size_t>(cells[n].t);
221 p = cb.globalProcStartPE[t];
222 for (std::size_t yproc = 0; yproc < static_cast<std::size_t>(cb.nprocy[t]); ++yproc) {
223 for (std::size_t xproc = 0; xproc < static_cast<std::size_t>(cb.nprocx[t]); ++xproc) {
224 if ((cells[n].y >= cb.yoffset[t][yproc]) && (cells[n].y < cb.yoffset[t][yproc + 1]) &&
225 (cells[n].x >= cb.xoffset[t][xproc]) && (cells[n].x < cb.xoffset[t][xproc + 1])) {
226 part[n] = p;
227 }
228 ++p;
229 }
230 }
231 }
232 ATLAS_ASSERT(part[nb_cells - 1] == nb_partitions() - 1, "number of partitions created not what is expected");
233 }
234
235 void CubedSpherePartitioner::partition(const Grid& grid, int part[]) const {
236 if (nb_partitions() == 1) // trivial solution, so much faster
237 {
238 for (idx_t j = 0; j < grid.size(); ++j) {
239 part[j] = 0;
240 }
241 }
242 else {
243 auto cb = cubedsphere(grid);
244
245 std::vector<CellInt> cells(static_cast<std::size_t>(grid.size()));
246 std::size_t n(0);
247
248 for (std::size_t it = 0; it < 6; ++it) {
249 for (idx_t iy = 0; iy < cb.ny[it]; ++iy) {
250 for (idx_t ix = 0; ix < cb.nx[it]; ++ix) {
251 cells[n].t = static_cast<int>(it);
252 cells[n].x = static_cast<int>(ix);
253 cells[n].y = static_cast<int>(iy);
254 cells[n].n = static_cast<int>(n);
255 ++n;
256 }
257 }
258 }
259 partition(cb, grid.size(), cells.data(), part);
260 }
261 }
262
263 } // namespace partitioner
264 } // namespace detail
265 } // namespace grid
266 } // namespace atlas
267
268 namespace {
269 atlas::grid::detail::partitioner::PartitionerBuilder<atlas::grid::detail::partitioner::CubedSpherePartitioner>
270 __CubedSphere("cubedsphere");
271 }