| 1 |
/* |
| 2 |
* (C) Crown Copyright 2021 Met Office |
| 3 |
* |
| 4 |
* This software is licensed under the terms of the Apache Licence Version 2.0 |
| 5 |
* which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. |
| 6 |
* In applying this licence, ECMWF does not waive the privileges and immunities |
| 7 |
* granted to it by virtue of its status as an intergovernmental organisation |
| 8 |
* nor does it submit to any jurisdiction. |
| 9 |
*/ |
| 10 |
|
| 11 |
|
| 12 |
#include "CubedSpherePartitioner.h" |
| 13 |
|
| 14 |
#include <algorithm> |
| 15 |
#include <cmath> |
| 16 |
#include <ctime> |
| 17 |
#include <functional> |
| 18 |
#include <iostream> |
| 19 |
#include <vector> |
| 20 |
|
| 21 |
#include "atlas/grid/CubedSphereGrid.h" |
| 22 |
#include "atlas/runtime/Exception.h" |
| 23 |
#include "atlas/runtime/Log.h" |
| 24 |
|
| 25 |
namespace atlas { |
| 26 |
namespace grid { |
| 27 |
namespace detail { |
| 28 |
namespace partitioner { |
| 29 |
|
| 30 |
namespace { |
| 31 |
|
| 32 |
bool isNearInt(double value) { |
| 33 |
const double diff = value - std::floor(value); |
| 34 |
return (diff <= std::numeric_limits<double>::epsilon() || diff >= (1.0 - std::numeric_limits<double>::epsilon())); |
| 35 |
} |
| 36 |
|
| 37 |
bool isConfigSufficient(const eckit::Parametrisation& config) { |
| 38 |
return !(config.has("starting rank on tile")) || !(config.has("final rank on tile")) || !(config.has("nprocx")) || |
| 39 |
!(config.has("nprocy")); |
| 40 |
} |
| 41 |
|
| 42 |
std::vector<std::vector<atlas::idx_t>> createOffset(const std::array<std::size_t, 6>& ngpt, |
| 43 |
const std::array<idx_t, 6> nproc1D, |
| 44 |
const std::array<idx_t, 6> maxDim1D, |
| 45 |
const std::array<idx_t, 6> maxDim2D) { |
| 46 |
std::vector<std::vector<atlas::idx_t>> offset; |
| 47 |
|
| 48 |
idx_t totalNproc1D = std::accumulate(std::begin(nproc1D), std::end(nproc1D), 0); |
| 49 |
|
| 50 |
// ngpbt - number of grid points per band and tile. |
| 51 |
std::vector<std::size_t> ngpbt(static_cast<std::size_t>(totalNproc1D), 0); |
| 52 |
|
| 53 |
idx_t j(0); |
| 54 |
for (std::size_t t = 0; t < 6; ++t) { |
| 55 |
idx_t jt = static_cast<idx_t>(j); |
| 56 |
std::size_t nproc = static_cast<std::size_t>(nproc1D[t]); |
| 57 |
std::vector<idx_t> offsetPerTile(nproc + 1, 0); |
| 58 |
|
| 59 |
for (std::size_t proc1D = 0; proc1D < nproc; ++proc1D, ++j) { |
| 60 |
ngpbt[static_cast<size_t>(j)] = |
| 61 |
proc1D < nproc - 1 |
| 62 |
? ngpt[t] / nproc |
| 63 |
: ngpt[t] - static_cast<std::size_t>(std::accumulate(ngpbt.begin() + jt, ngpbt.begin() + j, 0)); |
| 64 |
|
| 65 |
offsetPerTile[proc1D] = |
| 66 |
proc1D == 0 ? 0 : std::accumulate(ngpbt.begin() + jt, ngpbt.begin() + j, 0) / maxDim2D[t]; |
| 67 |
} |
| 68 |
offsetPerTile[nproc] = maxDim1D[t]; |
| 69 |
offset.push_back(offsetPerTile); |
| 70 |
} |
| 71 |
|
| 72 |
return offset; |
| 73 |
} |
| 74 |
|
| 75 |
} // namespace |
| 76 |
|
| 77 |
CubedSpherePartitioner::CubedSpherePartitioner(): Partitioner() {} |
| 78 |
|
| 79 |
CubedSpherePartitioner::CubedSpherePartitioner(int N): Partitioner(N, util::NoConfig()), regular_{true} {} |
| 80 |
|
| 81 |
CubedSpherePartitioner::CubedSpherePartitioner(const eckit::Parametrisation& config): |
| 82 |
Partitioner(config), regular_{isConfigSufficient(config)} { |
| 83 |
if (config.has("starting rank on tile") && config.has("final rank on tile") && config.has("nprocx") && |
| 84 |
config.has("nprocy")) { |
| 85 |
config.get("starting rank on tile", globalProcStartPE_); |
| 86 |
config.get("final rank on tile", globalProcEndPE_); |
| 87 |
config.get("nprocx", nprocx_); |
| 88 |
config.get("nprocy", nprocy_); |
| 89 |
} |
| 90 |
} |
| 91 |
|
| 92 |
CubedSpherePartitioner::CubedSpherePartitioner(int N, const eckit::Parametrisation& config): |
| 93 |
Partitioner(N, config), regular_{isConfigSufficient(config)} { |
| 94 |
if (config.has("starting rank on tile") && config.has("final rank on tile") && config.has("nprocx") && |
| 95 |
config.has("nprocy")) { |
| 96 |
config.get("starting rank on tile", globalProcStartPE_); |
| 97 |
config.get("final rank on tile", globalProcEndPE_); |
| 98 |
config.get("nprocx", nprocx_); |
| 99 |
config.get("nprocy", nprocy_); |
| 100 |
} |
| 101 |
} |
| 102 |
|
| 103 |
CubedSpherePartitioner::CubedSpherePartitioner(const int N, const std::vector<int>& globalProcStartPE, |
| 104 |
const std::vector<int>& globalProcEndPE, const std::vector<int>& nprocx, |
| 105 |
const std::vector<int>& nprocy): |
| 106 |
Partitioner(N, util::NoConfig()), |
| 107 |
globalProcStartPE_(globalProcStartPE.begin(), globalProcStartPE.end()), |
| 108 |
globalProcEndPE_(globalProcEndPE.begin(), globalProcEndPE.end()), |
| 109 |
nprocx_{nprocx.begin(), nprocx.end()}, |
| 110 |
nprocy_{nprocy.begin(), nprocy.end()}, |
| 111 |
regular_{false} {} |
| 112 |
|
| 113 |
|
| 114 |
CubedSpherePartitioner::CubedSphere CubedSpherePartitioner::cubedsphere(const Grid& grid) const { |
| 115 |
// grid dimensions |
| 116 |
const CubedSphereGrid cg(grid); |
| 117 |
if (!cg) { |
| 118 |
throw_Exception("CubedSphere Partitioner only works for cubed sphere grids.", Here()); |
| 119 |
} |
| 120 |
|
| 121 |
CubedSphere cb; |
| 122 |
|
| 123 |
for (std::size_t t = 0; t < 6; ++t) { |
| 124 |
cb.nx[t] = cg.N(); |
| 125 |
cb.ny[t] = cg.N(); |
| 126 |
} |
| 127 |
|
| 128 |
atlas::idx_t nparts = nb_partitions(); |
| 129 |
|
| 130 |
if (regular_) { |
| 131 |
// share PEs around tiles |
| 132 |
// minRanksPerTile |
| 133 |
|
| 134 |
idx_t ranksPerTile = nparts / 6; |
| 135 |
|
| 136 |
idx_t reminder = nparts - 6 * ranksPerTile; |
| 137 |
|
| 138 |
for (std::size_t t = 0; t < 6; ++t) { |
| 139 |
cb.nproc[t] = ranksPerTile; |
| 140 |
} |
| 141 |
|
| 142 |
// round-robin; |
| 143 |
std::size_t t{0}; |
| 144 |
while (reminder > 0) { |
| 145 |
if (t == 6) { |
| 146 |
t = 0; |
| 147 |
} |
| 148 |
cb.nproc[t] += 1; |
| 149 |
t += 1; |
| 150 |
reminder -= 1; |
| 151 |
} |
| 152 |
|
| 153 |
// now need to specify nprocx and nprocy. |
| 154 |
// nproc is 0 for the tile we use default nprocx and nprocy = 1 |
| 155 |
|
| 156 |
// if we can square-root nproc and get an integer |
| 157 |
// we use that for nprocx and nprocy |
| 158 |
// otherwise we split just in nprocx and keep nprocy =1; |
| 159 |
|
| 160 |
for (std::size_t t = 0; t < 6; ++t) { |
| 161 |
if (cb.nproc[t] > 0) { |
| 162 |
double sq = std::sqrt(static_cast<double>(cb.nproc[t])); |
| 163 |
if (isNearInt(sq)) { |
| 164 |
cb.nprocx[t] = static_cast<idx_t>(std::round(sq)); |
| 165 |
cb.nprocy[t] = static_cast<idx_t>(std::round(sq)); |
| 166 |
} |
| 167 |
else { |
| 168 |
cb.nprocx[t] = 1; |
| 169 |
cb.nprocy[t] = cb.nproc[t]; |
| 170 |
} |
| 171 |
} |
| 172 |
} |
| 173 |
cb.globalProcStartPE[0] = 0; |
| 174 |
cb.globalProcEndPE[0] = cb.nproc[0] - 1; |
| 175 |
|
| 176 |
for (size_t t = 1; t < 6; ++t) { |
| 177 |
if (cb.nproc[t] == 0) { |
| 178 |
cb.globalProcStartPE[t] = cb.globalProcEndPE[t - 1]; |
| 179 |
cb.globalProcEndPE[t] = cb.globalProcEndPE[t - 1]; |
| 180 |
} |
| 181 |
else { |
| 182 |
cb.globalProcStartPE[t] = cb.globalProcEndPE[t - 1] + 1; |
| 183 |
cb.globalProcEndPE[t] = cb.globalProcStartPE[t] + cb.nproc[t] - 1; |
| 184 |
} |
| 185 |
} |
| 186 |
} |
| 187 |
else { |
| 188 |
for (size_t t = 0; t < 6; ++t) { |
| 189 |
cb.globalProcStartPE[t] = globalProcStartPE_[t]; |
| 190 |
cb.globalProcEndPE[t] = globalProcEndPE_[t]; |
| 191 |
cb.nprocx[t] = nprocx_[t]; |
| 192 |
cb.nprocy[t] = nprocy_[t]; |
| 193 |
} |
| 194 |
cb.nproc[0] = globalProcEndPE_[0] + 1; |
| 195 |
for (std::size_t t = 1; t < 6; ++t) { |
| 196 |
cb.nproc[t] = cb.globalProcStartPE[t] == cb.globalProcEndPE[t - 1] |
| 197 |
? cb.nproc[t] = 0 |
| 198 |
: cb.globalProcEndPE[t] - cb.globalProcStartPE[t] + 1; |
| 199 |
} |
| 200 |
} |
| 201 |
return cb; |
| 202 |
} |
| 203 |
|
| 204 |
void CubedSpherePartitioner::partition(CubedSphere& cb, const int nb_cells, const CellInt cells[], int part[]) const { |
| 205 |
// ngpt number of grid points per tile (this is for cell centers where the number of cells are the same |
| 206 |
std::size_t tileCells = static_cast<std::size_t>(nb_cells / 6); |
| 207 |
std::array<std::size_t, 6> ngpt{tileCells, tileCells, tileCells, tileCells, tileCells, tileCells}; |
| 208 |
|
| 209 |
// xoffset - the xoffset per band per tile (including cb.nx[t]) |
| 210 |
//std::vector<std::vector<atlas::idx_t>> xoffset(createOffset(ngpt, cb.nprocx, cb.nx, cb.ny)); |
| 211 |
|
| 212 |
cb.xoffset = createOffset(ngpt, cb.nprocx, cb.nx, cb.ny); |
| 213 |
|
| 214 |
// yoffset - the yoffset per band per tile (including cb.ny[t]) |
| 215 |
cb.yoffset = createOffset(ngpt, cb.nprocy, cb.ny, cb.nx); |
| 216 |
|
| 217 |
int p; // partition index |
| 218 |
// loop over all data tile by tile |
| 219 |
for (int n = 0; n < nb_cells; ++n) { |
| 220 |
std::size_t t = static_cast<std::size_t>(cells[n].t); |
| 221 |
p = cb.globalProcStartPE[t]; |
| 222 |
for (std::size_t yproc = 0; yproc < static_cast<std::size_t>(cb.nprocy[t]); ++yproc) { |
| 223 |
for (std::size_t xproc = 0; xproc < static_cast<std::size_t>(cb.nprocx[t]); ++xproc) { |
| 224 |
if ((cells[n].y >= cb.yoffset[t][yproc]) && (cells[n].y < cb.yoffset[t][yproc + 1]) && |
| 225 |
(cells[n].x >= cb.xoffset[t][xproc]) && (cells[n].x < cb.xoffset[t][xproc + 1])) { |
| 226 |
part[n] = p; |
| 227 |
} |
| 228 |
++p; |
| 229 |
} |
| 230 |
} |
| 231 |
} |
| 232 |
ATLAS_ASSERT(part[nb_cells - 1] == nb_partitions() - 1, "number of partitions created not what is expected"); |
| 233 |
} |
| 234 |
|
| 235 |
void CubedSpherePartitioner::partition(const Grid& grid, int part[]) const { |
| 236 |
if (nb_partitions() == 1) // trivial solution, so much faster |
| 237 |
{ |
| 238 |
for (idx_t j = 0; j < grid.size(); ++j) { |
| 239 |
part[j] = 0; |
| 240 |
} |
| 241 |
} |
| 242 |
else { |
| 243 |
auto cb = cubedsphere(grid); |
| 244 |
|
| 245 |
std::vector<CellInt> cells(static_cast<std::size_t>(grid.size())); |
| 246 |
std::size_t n(0); |
| 247 |
|
| 248 |
for (std::size_t it = 0; it < 6; ++it) { |
| 249 |
for (idx_t iy = 0; iy < cb.ny[it]; ++iy) { |
| 250 |
for (idx_t ix = 0; ix < cb.nx[it]; ++ix) { |
| 251 |
cells[n].t = static_cast<int>(it); |
| 252 |
cells[n].x = static_cast<int>(ix); |
| 253 |
cells[n].y = static_cast<int>(iy); |
| 254 |
cells[n].n = static_cast<int>(n); |
| 255 |
++n; |
| 256 |
} |
| 257 |
} |
| 258 |
} |
| 259 |
partition(cb, grid.size(), cells.data(), part); |
| 260 |
} |
| 261 |
} |
| 262 |
|
| 263 |
} // namespace partitioner |
| 264 |
} // namespace detail |
| 265 |
} // namespace grid |
| 266 |
} // namespace atlas |
| 267 |
|
| 268 |
namespace { |
| 269 |
atlas::grid::detail::partitioner::PartitionerBuilder<atlas::grid::detail::partitioner::CubedSpherePartitioner> |
| 270 |
__CubedSphere("cubedsphere"); |
| 271 |
} |