| 1 |
/* |
| 2 |
* (C) Copyright 2013 ECMWF. |
| 3 |
* |
| 4 |
* This software is licensed under the terms of the Apache Licence Version 2.0 |
| 5 |
* which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. |
| 6 |
* In applying this licence, ECMWF does not waive the privileges and immunities |
| 7 |
* granted to it by virtue of its status as an intergovernmental organisation |
| 8 |
* nor does it submit to any jurisdiction. |
| 9 |
*/ |
| 10 |
|
| 11 |
#include <algorithm> |
| 12 |
#include <cmath> |
| 13 |
#include <iomanip> |
| 14 |
#include <limits> |
| 15 |
#include <numeric> |
| 16 |
#include <vector> |
| 17 |
|
| 18 |
#include "eckit/utils/Hash.h" |
| 19 |
|
| 20 |
#include "atlas/array/Array.h" |
| 21 |
#include "atlas/array/ArrayView.h" |
| 22 |
#include "atlas/array/IndexView.h" |
| 23 |
#include "atlas/field/Field.h" |
| 24 |
#include "atlas/grid/Distribution.h" |
| 25 |
#include "atlas/grid/Partitioner.h" |
| 26 |
#include "atlas/grid/StructuredGrid.h" |
| 27 |
#include "atlas/library/config.h" |
| 28 |
#include "atlas/mesh/ElementType.h" |
| 29 |
#include "atlas/mesh/Elements.h" |
| 30 |
#include "atlas/mesh/HybridElements.h" |
| 31 |
#include "atlas/mesh/Mesh.h" |
| 32 |
#include "atlas/mesh/Nodes.h" |
| 33 |
#include "atlas/meshgenerator/detail/HealpixMeshGenerator.h" |
| 34 |
#include "atlas/meshgenerator/detail/MeshGeneratorFactory.h" |
| 35 |
#include "atlas/parallel/mpi/mpi.h" |
| 36 |
#include "atlas/runtime/Exception.h" |
| 37 |
#include "atlas/runtime/Log.h" |
| 38 |
#include "atlas/util/CoordinateEnums.h" |
| 39 |
#include "atlas/util/Topology.h" |
| 40 |
|
| 41 |
#define DEBUG_OUTPUT 0 |
| 42 |
#define DEBUG_OUTPUT_DETAIL 0 |
| 43 |
|
| 44 |
using atlas::util::Topology; |
| 45 |
|
| 46 |
namespace atlas { |
| 47 |
namespace meshgenerator { |
| 48 |
|
| 49 |
HealpixMeshGenerator::HealpixMeshGenerator(const eckit::Parametrisation& p) { |
| 50 |
std::string mpi_comm = mpi::comm().name(); |
| 51 |
p.get("mpi_comm", mpi_comm); |
| 52 |
options.set("mpi_comm",mpi_comm); |
| 53 |
|
| 54 |
configure_defaults(); |
| 55 |
|
| 56 |
size_t nb_parts; |
| 57 |
if (p.get("nb_parts", nb_parts)) { |
| 58 |
options.set("nb_parts", nb_parts); |
| 59 |
} |
| 60 |
|
| 61 |
size_t part; |
| 62 |
if (p.get("part", part)) { |
| 63 |
options.set("part", part); |
| 64 |
} |
| 65 |
|
| 66 |
bool three_dimensional; |
| 67 |
if (p.get("3d", three_dimensional)) { |
| 68 |
options.set("3d", three_dimensional); |
| 69 |
} |
| 70 |
options.get("3d",three_dimensional); |
| 71 |
|
| 72 |
std::string pole_elements{ three_dimensional ? "quads" : options.getString("pole_elements") }; |
| 73 |
if (p.get("pole_elements", pole_elements)) { |
| 74 |
if (pole_elements != "pentagons" and pole_elements != "quads") { |
| 75 |
Log::warning() << "Atlas::HealpixMeshGenerator accepts \"pentagons\" or \"quads\" for \"pole_elements\"." |
| 76 |
<< "Defaulting to pole_elements = quads" << std::endl; |
| 77 |
} |
| 78 |
} |
| 79 |
options.set("pole_elements", pole_elements); |
| 80 |
|
| 81 |
std::string partitioner; |
| 82 |
if (p.get("partitioner", partitioner)) { |
| 83 |
if (not grid::Partitioner::exists(partitioner)) { |
| 84 |
Log::warning() << "Atlas does not have support for partitioner " << partitioner << ". " |
| 85 |
<< "Defaulting to use partitioner EqualRegions" << std::endl; |
| 86 |
partitioner = "equal_regions"; |
| 87 |
} |
| 88 |
options.set("partitioner", partitioner); |
| 89 |
} |
| 90 |
} |
| 91 |
|
| 92 |
void HealpixMeshGenerator::configure_defaults() { |
| 93 |
std::string mpi_comm; |
| 94 |
options.get("mpi_comm",mpi_comm); |
| 95 |
auto& comm = mpi::comm(mpi_comm); |
| 96 |
|
| 97 |
// This option sets number of parts the mesh will be split in |
| 98 |
options.set("nb_parts", comm.size()); |
| 99 |
|
| 100 |
// This option sets the part that will be generated |
| 101 |
options.set("part", comm.rank()); |
| 102 |
|
| 103 |
// This option switches between original HEALPix with 1 point at the pole (3d -> true) |
| 104 |
// or HEALPix with 8 or 4 points at the pole (3d -> false) |
| 105 |
options.set("3d", false); |
| 106 |
|
| 107 |
// This options switches between pentagons and quads as the pole elements for (3d -> false) |
| 108 |
// choose: "quads" or "pentagons" |
| 109 |
options.set("pole_elements", "pentagons"); |
| 110 |
|
| 111 |
// This options sets the default partitioner |
| 112 |
std::string partitioner; |
| 113 |
if (grid::Partitioner::exists("equal_regions") && comm.size() > 1) { |
| 114 |
partitioner = "equal_regions"; |
| 115 |
} |
| 116 |
else { |
| 117 |
partitioner = "serial"; |
| 118 |
} |
| 119 |
options.set<std::string>("partitioner", partitioner); |
| 120 |
} |
| 121 |
|
| 122 |
|
| 123 |
// match glb_idx of node in (nb_pole_nodes==8 and ==4)-meshes to glb_idx of nodes in (nb_pole_nodes==1)-mesh |
| 124 |
gidx_t HealpixMeshGenerator::match_node_idx(const gidx_t& gidx, const int ns) const { |
| 125 |
const gidx_t nb_nodes_orig = 12 * ns * ns; |
| 126 |
if (gidx > nb_nodes_ - 1) { |
| 127 |
// no change in index for the periodic nodes |
| 128 |
return gidx; |
| 129 |
} |
| 130 |
if (nb_pole_nodes_ > 1) { |
| 131 |
if (gidx == nb_pole_nodes_ / 2) { |
| 132 |
return 0; |
| 133 |
} |
| 134 |
if (gidx == nb_nodes_ - nb_pole_nodes_ / 2) { |
| 135 |
return nb_nodes_orig + 1; |
| 136 |
} |
| 137 |
bool at_north_pole = (gidx < nb_pole_nodes_); |
| 138 |
bool at_south_pole = (gidx < nb_nodes_ and gidx >= nb_nodes_orig + nb_pole_nodes_); |
| 139 |
if (at_north_pole) { |
| 140 |
return nb_nodes_orig + 2 + gidx - (gidx > nb_pole_nodes_ / 2 ? 1 : 0); |
| 141 |
} |
| 142 |
if (at_south_pole) { |
| 143 |
return gidx - nb_pole_nodes_ + nb_pole_nodes_ + 1 - (gidx > nb_nodes_orig + nb_pole_nodes_ * 3 / 2 ? 1 : 0); |
| 144 |
} |
| 145 |
return gidx - nb_pole_nodes_ + 1; |
| 146 |
} |
| 147 |
// no change for 3d healpix mesh with one node per pole, i.e. nb_pole_nodes = 1 |
| 148 |
return gidx; |
| 149 |
} |
| 150 |
|
| 151 |
|
| 152 |
// return "global_id - 1" |
| 153 |
gidx_t HealpixMeshGenerator::idx_xy_to_x(const int xidx, const int yidx, const int ns) const { |
| 154 |
ATLAS_ASSERT(yidx < 4 * ns + 1 && yidx >= 0); |
| 155 |
ATLAS_ASSERT(xidx >= 0); |
| 156 |
|
| 157 |
const gidx_t nb_nodes_orig = 12 * ns * ns; |
| 158 |
auto ghostIdx = [this](int latid) { return this->nb_nodes_ + latid; }; |
| 159 |
gidx_t ret; |
| 160 |
|
| 161 |
if (yidx == 0) { |
| 162 |
ATLAS_ASSERT(xidx <= nb_pole_nodes_ && xidx >= 0); |
| 163 |
ret = (xidx != nb_pole_nodes_ ? xidx : ghostIdx(yidx)); |
| 164 |
} |
| 165 |
else if (yidx < ns) { |
| 166 |
ATLAS_ASSERT(xidx < 4 * yidx + 1 && xidx >= 0); |
| 167 |
ret = (xidx != 4 * yidx ? 2 * yidx * (yidx - 1) + nb_pole_nodes_ + xidx : ghostIdx(yidx)); |
| 168 |
} |
| 169 |
else if (yidx <= 2 * ns) { |
| 170 |
ATLAS_ASSERT(xidx < 4 * ns + 1 && xidx >= 0); |
| 171 |
ret = (xidx != 4 * ns ? 2 * ns * (ns - 1) + 4 * ns * (yidx - ns) + nb_pole_nodes_ + xidx : ghostIdx(yidx)); |
| 172 |
} |
| 173 |
else if (yidx <= 3 * ns) { |
| 174 |
ATLAS_ASSERT(xidx < 4 * ns + 1 && xidx >= 0); |
| 175 |
ret = (xidx != 4 * ns ? 2 * ns * (3 * ns + 1) + 4 * ns * (yidx - 2 * ns - 1) + nb_pole_nodes_ + xidx |
| 176 |
: ghostIdx(yidx)); |
| 177 |
} |
| 178 |
else if (yidx == 3 * ns + 1 && ns > 1) { |
| 179 |
ATLAS_ASSERT(xidx < 4 * (ns - 1) + 1 && xidx >= 0); |
| 180 |
ret = (xidx != 4 * (ns - 1) ? 2 * ns * (5 * ns + 1) + 4 * ns * (yidx - 3 * ns - 1) + nb_pole_nodes_ + xidx |
| 181 |
: ghostIdx(yidx)); |
| 182 |
} |
| 183 |
else if (yidx < 4 * ns) { |
| 184 |
ATLAS_ASSERT(xidx < 4 * (ns - (yidx - 3 * ns)) + 1 && xidx >= 0); |
| 185 |
ret = |
| 186 |
(xidx != 4 * (ns - (yidx - 3 * ns)) ? 2 * ns * (5 * ns + 1) + 4 * ns * (yidx - 3 * ns - 1) - |
| 187 |
2 * (yidx - 3 * ns) * (yidx - 3 * ns - 1) + nb_pole_nodes_ + xidx |
| 188 |
: ghostIdx(yidx)); |
| 189 |
} |
| 190 |
else { |
| 191 |
ATLAS_ASSERT(xidx <= nb_pole_nodes_ && xidx >= 0); |
| 192 |
ret = (xidx != nb_pole_nodes_ ? nb_nodes_orig + nb_pole_nodes_ + xidx : ghostIdx(yidx)); |
| 193 |
} |
| 194 |
return ret; |
| 195 |
} |
| 196 |
|
| 197 |
// return global_id of the node "above" (xidx,yidx) node |
| 198 |
gidx_t HealpixMeshGenerator::up_idx(const int xidx, const int yidx, const int ns) const { |
| 199 |
ATLAS_ASSERT(yidx <= 4 * ns && yidx >= 0); |
| 200 |
|
| 201 |
const gidx_t nb_nodes_orig = 12 * ns * ns; |
| 202 |
auto ghostIdx = [this](int latid) { return this->nb_nodes_ + latid; }; |
| 203 |
|
| 204 |
int ret; |
| 205 |
|
| 206 |
if (yidx == 0) { |
| 207 |
ATLAS_ASSERT(xidx < nb_pole_nodes_); |
| 208 |
ret = (xidx != nb_pole_nodes_ - 1 ? xidx + 1 : ghostIdx(0)); |
| 209 |
} |
| 210 |
else if (yidx == 1) { |
| 211 |
ATLAS_ASSERT(xidx < 4); |
| 212 |
ret = (nb_pole_nodes_ == 8 ? 2 * xidx : (nb_pole_nodes_ == 4 ? xidx : 0)); |
| 213 |
} |
| 214 |
else if (yidx < ns) { |
| 215 |
ATLAS_ASSERT(xidx < 4 * yidx); |
| 216 |
if (xidx != 4 * yidx - 1) { |
| 217 |
ret = 2 * (yidx - 2) * (yidx - 1) + nb_pole_nodes_ + xidx - std::floor(xidx / (double)yidx); |
| 218 |
} |
| 219 |
else { |
| 220 |
ret = ghostIdx(yidx - 1); |
| 221 |
} |
| 222 |
} |
| 223 |
else if (yidx == ns && ns < 3) { |
| 224 |
ATLAS_ASSERT(xidx < 4 * ns); |
| 225 |
if (xidx != 4 * ns - 1) { |
| 226 |
ret = 2 * ns * (ns - 1) + nb_pole_nodes_ - 4 * (ns - 1) + (xidx + 1) / 2; |
| 227 |
} |
| 228 |
else { |
| 229 |
ret = ghostIdx(yidx - 1); |
| 230 |
} |
| 231 |
} |
| 232 |
else if (yidx == ns) { |
| 233 |
ATLAS_ASSERT(xidx < 4 * ns); |
| 234 |
if (xidx != 4 * ns - 1) { |
| 235 |
ret = 2 * (ns - 2) * (ns - 1) + nb_pole_nodes_ + xidx - std::floor(xidx / (double)yidx); |
| 236 |
} |
| 237 |
else { |
| 238 |
ret = ghostIdx(yidx - 1); |
| 239 |
} |
| 240 |
} |
| 241 |
else if (yidx <= 3 * ns) { |
| 242 |
ATLAS_ASSERT(xidx < 4 * ns); |
| 243 |
int staggering = (yidx - ns) % 2; |
| 244 |
if (xidx != 4 * ns - 1 || (xidx == 4 * ns - 1 && staggering)) { |
| 245 |
ret = 2 * ns * (ns - 1) + nb_pole_nodes_ + 4 * ns * (yidx - ns - 1) + xidx + 1 - staggering; |
| 246 |
} |
| 247 |
else { |
| 248 |
ret = ghostIdx(yidx - 1); |
| 249 |
} |
| 250 |
} |
| 251 |
else if (yidx < 4 * ns - 1) { |
| 252 |
int yidxl = 4 * ns - yidx; |
| 253 |
ATLAS_ASSERT(xidx < 4 * yidxl); |
| 254 |
ret = nb_nodes_orig + nb_pole_nodes_ + 1 - 2 * (yidxl + 2) * (yidxl + 1) + xidx + |
| 255 |
std::floor(xidx / (double)yidxl); |
| 256 |
} |
| 257 |
else if (yidx == 4 * ns - 1) { |
| 258 |
ATLAS_ASSERT(xidx < 4); |
| 259 |
ret = nb_nodes_orig + nb_pole_nodes_ - 3 - (ns == 1 ? 4 : 8) + 2 * xidx; |
| 260 |
} |
| 261 |
else { |
| 262 |
ATLAS_ASSERT(xidx < nb_pole_nodes_); |
| 263 |
if (ns == 1) { |
| 264 |
if (xidx != nb_pole_nodes_ - 1) { |
| 265 |
ret = nb_nodes_orig + nb_pole_nodes_ - 4 + |
| 266 |
(nb_pole_nodes_ != 4 ? (xidx % 2 ? -4 + (xidx + 1) / 2 : xidx / 2) : xidx); |
| 267 |
} |
| 268 |
else { |
| 269 |
ret = (nb_pole_nodes_ == 4 ? nb_nodes_orig + xidx |
| 270 |
: (nb_pole_nodes_ == 8 ? ghostIdx(4 * ns - 2) : ghostIdx(4 * ns))); |
| 271 |
} |
| 272 |
} |
| 273 |
else { |
| 274 |
ret = nb_nodes_orig + nb_pole_nodes_ + (xidx % 2 ? xidx - 12 : xidx - 4 - xidx / 2); |
| 275 |
} |
| 276 |
} |
| 277 |
return ret; |
| 278 |
} |
| 279 |
|
| 280 |
// return global_id of the node "below" (xidx,yidx) node |
| 281 |
gidx_t HealpixMeshGenerator::down_idx(const int xidx, const int yidx, const int ns) const { |
| 282 |
ATLAS_ASSERT(yidx <= 4 * ns); |
| 283 |
|
| 284 |
const gidx_t nb_nodes_orig = 12 * ns * ns; |
| 285 |
auto ghostIdx = [this](int latid) { return this->nb_nodes_ + latid; }; |
| 286 |
|
| 287 |
int ret; |
| 288 |
|
| 289 |
if (yidx == 0) { |
| 290 |
ATLAS_ASSERT(xidx < nb_pole_nodes_); |
| 291 |
if (ns == 1) { |
| 292 |
if (xidx != nb_pole_nodes_ - 1) { |
| 293 |
if (nb_pole_nodes_ == 8) { |
| 294 |
ret = 8 + (xidx % 2 ? 4 + (xidx + 1) / 2 : xidx / 2); |
| 295 |
} |
| 296 |
else { |
| 297 |
ret = 1 + xidx; |
| 298 |
} |
| 299 |
} |
| 300 |
else { |
| 301 |
ret = ghostIdx(2); |
| 302 |
} |
| 303 |
} |
| 304 |
else { |
| 305 |
ret = nb_pole_nodes_ + ((xidx + 1) % 2 ? xidx / 2 : 4 + xidx); |
| 306 |
} |
| 307 |
} |
| 308 |
else if (yidx < ns) { |
| 309 |
ATLAS_ASSERT(xidx < 4 * yidx); |
| 310 |
ret = 2 * yidx * (yidx + 1) + nb_pole_nodes_ + 1 + xidx + std::floor(xidx / (double)yidx); |
| 311 |
} |
| 312 |
else if (yidx == ns && ns == 1) { |
| 313 |
ATLAS_ASSERT(xidx < 4); |
| 314 |
ret = (xidx != 3 ? nb_pole_nodes_ + 5 + xidx : ghostIdx(2)); |
| 315 |
} |
| 316 |
else if (yidx == 2 * ns && ns == 1) { |
| 317 |
ATLAS_ASSERT(xidx < 4); |
| 318 |
ret = (nb_pole_nodes_ == 8 ? 16 + xidx : (nb_pole_nodes_ == 4 ? 12 + xidx : 9 + xidx)); |
| 319 |
} |
| 320 |
else if (yidx < 3 * ns && ns > 1) { |
| 321 |
ATLAS_ASSERT(xidx < 4 * ns); |
| 322 |
int staggering = (yidx - ns) % 2; |
| 323 |
if (xidx != 4 * ns - 1 || (xidx == 4 * ns - 1 && staggering)) { |
| 324 |
ret = 2 * ns * (ns - 1) + nb_pole_nodes_ + 4 * ns * (yidx - ns + 1) + xidx + |
| 325 |
(yidx != 3 * ns ? 1 - staggering : 0); |
| 326 |
} |
| 327 |
else { |
| 328 |
ret = ghostIdx(yidx + 1); |
| 329 |
} |
| 330 |
} |
| 331 |
else if (yidx == 4 * ns - 2) { |
| 332 |
ATLAS_ASSERT(xidx < 8); |
| 333 |
if (nb_pole_nodes_ == 8) { |
| 334 |
ret = (xidx != 7 ? nb_nodes_orig + 4 + (xidx + 1) / 2 : ghostIdx(4 * ns - 1)); |
| 335 |
} |
| 336 |
else if (nb_pole_nodes_ == 4) { |
| 337 |
ret = (xidx != 7 ? nb_nodes_orig + (xidx + 1) / 2 : ghostIdx(4 * ns - 1)); |
| 338 |
} |
| 339 |
else { |
| 340 |
ret = (xidx != 7 ? nb_nodes_orig - 3 + (xidx + 1) / 2 : ghostIdx(4 * ns - 1)); |
| 341 |
} |
| 342 |
} |
| 343 |
else if (yidx == 4 * ns - 1) { |
| 344 |
ATLAS_ASSERT(xidx < 4); |
| 345 |
if (nb_pole_nodes_ == 8) { |
| 346 |
ret = nb_nodes_orig + nb_pole_nodes_ + 2 * xidx; |
| 347 |
} |
| 348 |
else if (nb_pole_nodes_ == 4) { |
| 349 |
ret = nb_nodes_orig + nb_pole_nodes_ + xidx; |
| 350 |
} |
| 351 |
else { |
| 352 |
ret = nb_nodes_orig + 1; |
| 353 |
} |
| 354 |
} |
| 355 |
else if (yidx < 4 * ns - 1) { |
| 356 |
int yidxl = yidx - 3 * ns; |
| 357 |
ATLAS_ASSERT(xidx < 4 * (ns - yidxl)); |
| 358 |
if (xidx != 4 * (ns - yidxl) - 1) { |
| 359 |
ret = 2 * ns * (5 * ns + 1) + nb_pole_nodes_ + 4 * ns * yidxl - 2 * (yidxl + 1) * yidxl + xidx - |
| 360 |
std::floor(xidx / (double)(ns - yidxl)); |
| 361 |
} |
| 362 |
else { |
| 363 |
ret = ghostIdx(yidx + 1); |
| 364 |
} |
| 365 |
} |
| 366 |
else if (yidx == 4 * ns) { |
| 367 |
ATLAS_ASSERT(xidx < nb_pole_nodes_); |
| 368 |
ret = (xidx != nb_pole_nodes_ - 1 ? nb_nodes_orig + nb_pole_nodes_ + xidx + 1 : ghostIdx(yidx)); |
| 369 |
} |
| 370 |
else { |
| 371 |
throw_AssertionFailed("Invalid value of yidx", Here()); |
| 372 |
} |
| 373 |
return ret; |
| 374 |
} |
| 375 |
|
| 376 |
// return global_id of the node "to the right of" (xidx,yidx) node |
| 377 |
gidx_t HealpixMeshGenerator::right_idx(const int xidx, const int yidx, const int ns) const { |
| 378 |
ATLAS_ASSERT(yidx <= 4 * ns); |
| 379 |
|
| 380 |
const gidx_t nb_nodes_orig = 12 * ns * ns; |
| 381 |
auto ghostIdx = [this](int latid) { return this->nb_nodes_ + latid; }; |
| 382 |
int ret = -1; |
| 383 |
|
| 384 |
if (yidx == 0) { |
| 385 |
ATLAS_ASSERT(xidx < nb_pole_nodes_); |
| 386 |
if (ns == 1) { |
| 387 |
ret = (xidx != nb_pole_nodes_ - 1 |
| 388 |
? (xidx % 2 ? nb_pole_nodes_ + (xidx + 1) / 2 : nb_pole_nodes_ + 5 + xidx / 2) |
| 389 |
: ghostIdx(1)); |
| 390 |
} |
| 391 |
else { |
| 392 |
ret = (xidx != nb_pole_nodes_ - 1 ? (xidx % 2 ? nb_pole_nodes_ + (xidx + 1) / 2 : nb_pole_nodes_ + 5 + xidx) |
| 393 |
: ghostIdx(1)); |
| 394 |
} |
| 395 |
} |
| 396 |
else if (yidx == 1) { |
| 397 |
ATLAS_ASSERT(xidx < 4); |
| 398 |
if (nb_pole_nodes_ == 8) { |
| 399 |
ret = 1 + 2 * xidx; |
| 400 |
} |
| 401 |
else if (nb_pole_nodes_ == 4) { |
| 402 |
ret = (xidx != 3 ? xidx + 1 : ghostIdx(0)); |
| 403 |
} |
| 404 |
else { |
| 405 |
ret = (xidx != 3 ? xidx + 2 : ghostIdx(1)); |
| 406 |
} |
| 407 |
} |
| 408 |
else if (yidx < ns) { |
| 409 |
ATLAS_ASSERT(xidx < 4 * yidx); |
| 410 |
ret = (xidx != 4 * yidx - 1 ? 2 * yidx * (yidx - 1) + nb_pole_nodes_ + 1 + xidx : ghostIdx(yidx)); |
| 411 |
} |
| 412 |
else if (yidx == 3 && ns == 1) { |
| 413 |
ATLAS_ASSERT(xidx < 4); |
| 414 |
if (nb_pole_nodes_ == 8) { |
| 415 |
ret = 21 + 2 * xidx; |
| 416 |
} |
| 417 |
else if (nb_pole_nodes_ == 4) { |
| 418 |
ret = (xidx != 3 ? 17 + xidx : ghostIdx(yidx + 1)); |
| 419 |
} |
| 420 |
else { |
| 421 |
ret = (xidx != 3 ? 10 + xidx : ghostIdx(yidx)); |
| 422 |
} |
| 423 |
} |
| 424 |
else if (yidx <= 3 * ns) { |
| 425 |
ATLAS_ASSERT(xidx < 4 * ns + 1); |
| 426 |
ret = (xidx != 4 * ns - 1 ? 2 * ns * (ns - 1) + 4 * ns * (yidx - ns) + nb_pole_nodes_ + 1 + xidx |
| 427 |
: ghostIdx(yidx)); |
| 428 |
} |
| 429 |
else if (yidx < 4 * ns - 1 && ns > 1) { |
| 430 |
int yidxl = yidx - 3 * ns; |
| 431 |
ATLAS_ASSERT(xidx < 4 * (ns - yidxl)); |
| 432 |
if (xidx != 4 * (ns - yidxl) - 1) { |
| 433 |
ret = 2 * ns * (5 * ns + 1) + 4 * ns * (yidx - 3 * ns - 1) - 2 * (yidx - 3 * ns) * (yidx - 3 * ns - 1) + |
| 434 |
nb_pole_nodes_ + 1 + xidx; |
| 435 |
} |
| 436 |
else { |
| 437 |
ret = ghostIdx(yidx); |
| 438 |
} |
| 439 |
} |
| 440 |
else if (yidx == 4 * ns - 1) { |
| 441 |
ATLAS_ASSERT(xidx < 4); |
| 442 |
if (nb_pole_nodes_ == 8) { |
| 443 |
ret = nb_nodes_orig + nb_pole_nodes_ + 1 + 2 * xidx; |
| 444 |
} |
| 445 |
else if (nb_pole_nodes_ == 4) { |
| 446 |
ret = (xidx != 3 ? nb_nodes_orig + nb_pole_nodes_ + 1 + xidx : ghostIdx(yidx + 1)); |
| 447 |
} |
| 448 |
else { |
| 449 |
ret = (xidx != 3 ? nb_nodes_orig - 2 + xidx : ghostIdx(yidx)); |
| 450 |
} |
| 451 |
} |
| 452 |
else if (yidx == 4 * ns) { |
| 453 |
ATLAS_ASSERT(xidx < nb_pole_nodes_); |
| 454 |
if (xidx != nb_pole_nodes_ - 1) { |
| 455 |
ret = (xidx % 2 ? nb_nodes_orig + 4 + (xidx + 1) / 2 : nb_nodes_orig + 4 - (ns == 1 ? 3 : 7) + xidx); |
| 456 |
} |
| 457 |
else { |
| 458 |
ret = ghostIdx(yidx - 1); |
| 459 |
} |
| 460 |
} |
| 461 |
return ret; |
| 462 |
} |
| 463 |
|
| 464 |
// return global_id - 1 of the pentagon node "to the right of" (xidx,yidx) node |
| 465 |
// pentagon points are only needed for yidx == 1 and yidx == 4 * ns - 1 |
| 466 |
gidx_t HealpixMeshGenerator::pentagon_right_idx(const int xidx, const int yidx, const int ns) const { |
| 467 |
auto ghostIdx = [this](int latid) { return this->nb_nodes_ + latid; }; |
| 468 |
if (yidx == 1) { |
| 469 |
return (xidx != 3 ? nb_pole_nodes_ + xidx + 1 : ghostIdx(1)); |
| 470 |
} |
| 471 |
else if (yidx == 4 * ns - 1) { |
| 472 |
return (xidx != 3 ? (12 * ns * ns + xidx + 1) : ghostIdx(yidx)); |
| 473 |
} |
| 474 |
else { |
| 475 |
return -2; |
| 476 |
} |
| 477 |
} |
| 478 |
|
| 479 |
void HealpixMeshGenerator::generate(const Grid& grid, Mesh& mesh) const { |
| 480 |
ATLAS_ASSERT(HealpixGrid(grid), "Grid could not be cast to a HealpixGrid"); |
| 481 |
ATLAS_ASSERT(!mesh.generated()); |
| 482 |
|
| 483 |
const StructuredGrid rg = StructuredGrid(grid); |
| 484 |
if (!rg) { |
| 485 |
throw_Exception("HealpixMeshGenerator can only work with a Healpix grid", Here()); |
| 486 |
} |
| 487 |
|
| 488 |
size_t nb_parts = options.get<size_t>("nb_parts"); |
| 489 |
|
| 490 |
std::string partitioner_type = "equal_regions"; |
| 491 |
options.get("partitioner", partitioner_type); |
| 492 |
|
| 493 |
mpi::scope::push(options.getString("mpi_comm")); |
| 494 |
grid::Partitioner partitioner(partitioner_type, nb_parts); |
| 495 |
grid::Distribution distribution(partitioner.partition(grid)); |
| 496 |
mpi::scope::pop(); |
| 497 |
generate(grid, distribution, mesh); |
| 498 |
} |
| 499 |
|
| 500 |
void HealpixMeshGenerator::hash(eckit::Hash& h) const { |
| 501 |
h.add("HealpixMeshGenerator"); |
| 502 |
options.hash(h); |
| 503 |
} |
| 504 |
|
| 505 |
void HealpixMeshGenerator::generate(const Grid& grid, const grid::Distribution& distribution, Mesh& mesh) const { |
| 506 |
ATLAS_TRACE(); |
| 507 |
Log::debug() << "HealpixMeshGenerator generating mesh from " << grid.name() << std::endl; |
| 508 |
ATLAS_ASSERT(HealpixGrid(grid), "Grid could not be cast to a HealpixGrid"); |
| 509 |
ATLAS_ASSERT(!mesh.generated()); |
| 510 |
|
| 511 |
if (grid.size() != static_cast<idx_t>(distribution.size())) { |
| 512 |
std::stringstream msg; |
| 513 |
msg << "Number of points in grid (" << grid.size() |
| 514 |
<< ") different from " |
| 515 |
"number of points in grid distribution (" |
| 516 |
<< distribution.size() << ")"; |
| 517 |
throw_AssertionFailed(msg.str(), Here()); |
| 518 |
} |
| 519 |
|
| 520 |
// clone some grid properties |
| 521 |
setGrid(mesh, grid, distribution); |
| 522 |
|
| 523 |
generate_mesh(grid, distribution, mesh); |
| 524 |
} |
| 525 |
|
| 526 |
void HealpixMeshGenerator::generate_mesh(const StructuredGrid& grid, const grid::Distribution& distribution, |
| 527 |
Mesh& mesh) const { |
| 528 |
// This function should do the following: |
| 529 |
// - define nodes with |
| 530 |
// mesh.nodes().resize(nnodes); |
| 531 |
// mesh::Nodes& nodes = mesh.nodes(); |
| 532 |
// following properties should be defined: |
| 533 |
// array::ArrayView<double,2> xy ( nodes.xy() ); |
| 534 |
// array::ArrayView<gidx_t,1> glb_idx ( nodes.global_index() ); |
| 535 |
// array::ArrayView<int, 1> part ( nodes.partition() ); |
| 536 |
// array::ArrayView<int, 1> ghost ( nodes.ghost() ); |
| 537 |
// array::ArrayView<int, 1> flags ( nodes.flags() ); |
| 538 |
// - define cells (only quadrilaterals for now) with |
| 539 |
// mesh.cells().add( mesh::ElementType::create("Quadrilateral"), nquads ); |
| 540 |
// mesh.cells().add( mesh::ElementType::create("Pentagon"), npents ); |
| 541 |
// further define cells with |
| 542 |
// array::ArrayView<gidx_t,1> cells_glb_idx( mesh.cells().global_index() |
| 543 |
// ); |
| 544 |
// array::ArrayView<int,1> cells_part( mesh.cells().partition() ); |
| 545 |
// - define connectivity with |
| 546 |
// mesh::HybridElements::Connectivity& node_connectivity = |
| 547 |
// mesh.cells().node_connectivity(); |
| 548 |
// node_connectivity.set( jcell, cell_nodes ); |
| 549 |
// where cell_nodes is a 4-element or 5-element integer array containing the LOCAL |
| 550 |
// indices of the nodes |
| 551 |
// |
| 552 |
// The rule do determine if a cell belongs to a proc is the following with some |
| 553 |
// exceptions near poles: cell belongs to a proc if the left corner of the cell belongs to that proc. |
| 554 |
|
| 555 |
|
| 556 |
ATLAS_TRACE(); |
| 557 |
|
| 558 |
ATLAS_ASSERT(HealpixGrid(grid)); |
| 559 |
|
| 560 |
const int mypart = options.get<size_t>("part"); |
| 561 |
const int nparts = options.get<size_t>("nb_parts"); |
| 562 |
const bool three_dimensional = options.get<bool>("3d"); |
| 563 |
const std::string pole_elements = options.get<std::string>("pole_elements"); |
| 564 |
const int nb_pole_nodes = (pole_elements == "pentagons") ? 4 : (three_dimensional ? 1 : 8); |
| 565 |
const int ny = grid.ny() + 2; |
| 566 |
const int ns = (ny - 1) / 4; |
| 567 |
const int nvertices = 12 * ns * ns + 2 * nb_pole_nodes; |
| 568 |
|
| 569 |
|
| 570 |
nb_pole_nodes_ = nb_pole_nodes; |
| 571 |
nb_points_ = 12 * ns * ns + (nb_pole_nodes == 8 ? 8 : 0); |
| 572 |
nb_nodes_ = nvertices; |
| 573 |
|
| 574 |
int inode; |
| 575 |
auto nb_lat_nodes = [ny, nb_pole_nodes, &grid](int latid) { |
| 576 |
return ((latid == 0) or (latid == ny - 1) ? nb_pole_nodes : grid.nx()[latid - 1]); |
| 577 |
}; |
| 578 |
|
| 579 |
int ii, ix, iy, ii_ghost, ii_glb; |
| 580 |
int iy_min, iy_max; // a belt (iy_min:iy_max) surrounding the nodes on this processor |
| 581 |
int nnodes_nonghost; // non-ghost node: belongs to this part |
| 582 |
|
| 583 |
// vector of local indices: necessary for remote indices of ghost nodes |
| 584 |
std::vector<int> local_idx(nvertices, -1); |
| 585 |
std::vector<int> current_idx(nparts, 0); // index counter for each proc |
| 586 |
|
| 587 |
// ANSATZ: requirement on the partitioner |
| 588 |
auto compute_part = [&](int iy, gidx_t ii_glb) -> int { |
| 589 |
// nodes at the pole belong to proc_0 (north) and proc_maxRank (south) |
| 590 |
// a node gets its proc rank from the element for which this node would be its west vertex |
| 591 |
return (iy == 0 ? 0 : (iy == ny - 1 ? nparts - 1 : distribution.partition(ii_glb - nb_pole_nodes))); |
| 592 |
}; |
| 593 |
|
| 594 |
#if DEBUG_OUTPUT_DETAIL |
| 595 |
for (iy = 0; iy < ny; iy++) { |
| 596 |
int nx = nb_lat_nodes(iy); |
| 597 |
for (ix = 0; ix < nx; ix++) { |
| 598 |
Log::info() << "iy, ix, glb_idx, up_idx, down_idx, right_idx, pent_right_idx : " << iy << ", " << ix << ", " |
| 599 |
<< idx_xy_to_x(ix, iy, ns) + 1 << ", " << up_idx(ix, iy, ns) + 1 << ", " |
| 600 |
<< down_idx(ix, iy, ns) + 1 << ", " << right_idx(ix, iy, ns) + 1 << ", " |
| 601 |
<< pentagon_right_idx(ix, iy, ns) + 1 << std::endl; |
| 602 |
} |
| 603 |
Log::info() << std::endl; |
| 604 |
} |
| 605 |
#endif |
| 606 |
|
| 607 |
// loop over all points to determine local indices and surrounding rectangle |
| 608 |
ii_glb = 0; // global index starting from 0 |
| 609 |
iy_min = ny + 1; |
| 610 |
iy_max = 0; |
| 611 |
nnodes_nonghost = 0; |
| 612 |
for (iy = 0; iy < ny; iy++) { |
| 613 |
int nx = nb_lat_nodes(iy); |
| 614 |
for (ix = 0; ix < nx; ix++) { |
| 615 |
int proc_id = compute_part(iy, ii_glb); |
| 616 |
local_idx[ii_glb] = current_idx[proc_id]++; |
| 617 |
if (proc_id == mypart) { |
| 618 |
++nnodes_nonghost; |
| 619 |
iy_min = std::min(iy_min, iy); |
| 620 |
iy_max = std::max(iy_max, iy); |
| 621 |
} |
| 622 |
++ii_glb; |
| 623 |
} |
| 624 |
} |
| 625 |
|
| 626 |
#if DEBUG_OUTPUT_DETAIL |
| 627 |
inode = 0; |
| 628 |
Log::info() << "local_idx : " << std::endl; |
| 629 |
for (size_t ilat = 0; ilat < ny; ilat++) { |
| 630 |
for (size_t ilon = 0; ilon < nb_lat_nodes(ilat); ilon++) { |
| 631 |
Log::info() << std::setw(4) << local_idx[inode]; |
| 632 |
inode++; |
| 633 |
} |
| 634 |
Log::info() << std::endl; |
| 635 |
} |
| 636 |
inode = 0; |
| 637 |
Log::info() << "global_idx : " << std::endl; |
| 638 |
for (size_t ilat = 0; ilat < ny; ilat++) { |
| 639 |
for (size_t ilon = 0; ilon < nb_lat_nodes(ilat); ilon++) { |
| 640 |
Log::info() << std::setw(4) << inode; |
| 641 |
inode++; |
| 642 |
} |
| 643 |
Log::info() << std::endl; |
| 644 |
} |
| 645 |
#endif |
| 646 |
|
| 647 |
// dimensions of surrounding belt (SB) |
| 648 |
int nnodes_SB = 0; |
| 649 |
if (iy_min <= 2) { |
| 650 |
iy_min = 0; |
| 651 |
} |
| 652 |
else { |
| 653 |
--iy_min; |
| 654 |
} |
| 655 |
if (iy_max >= ny - 3) { |
| 656 |
iy_max = ny - 1; |
| 657 |
} |
| 658 |
else { |
| 659 |
++iy_max; |
| 660 |
} |
| 661 |
for (int iy = iy_min; iy <= iy_max; iy++) { |
| 662 |
// (east) periodic point adds +1 here |
| 663 |
nnodes_SB += nb_lat_nodes(iy) + 1; |
| 664 |
} |
| 665 |
|
| 666 |
#if DEBUG_OUTPUT |
| 667 |
Log::info() << "[" << mypart << "] : nnodes_SB = " << nnodes_SB << "\n"; |
| 668 |
#endif |
| 669 |
|
| 670 |
// partitions and local indices in SB |
| 671 |
std::vector<int> parts_SB(nnodes_SB, -1); |
| 672 |
std::vector<int> local_idx_SB(nnodes_SB, -1); |
| 673 |
std::vector<bool> is_ghost_SB(nnodes_SB, true); |
| 674 |
|
| 675 |
// starting from index 0, first global node-index for this partition |
| 676 |
int parts_sidx = idx_xy_to_x(0, iy_min, ns); |
| 677 |
|
| 678 |
ii = 0; // index inside SB |
| 679 |
ii_ghost = nnodes_SB - (iy_max - iy_min + 1); // first local ghost idx |
| 680 |
for (iy = iy_min; iy <= iy_max; iy++) { |
| 681 |
int nx = nb_lat_nodes(iy) + 1; |
| 682 |
for (ix = 0; ix < nx; ix++) { |
| 683 |
if (ix != nx - 1) { |
| 684 |
ii_glb = ii + parts_sidx; |
| 685 |
parts_SB[ii] = compute_part(iy, ii_glb); |
| 686 |
local_idx_SB[ii] = ii; |
| 687 |
is_ghost_SB[ii] = !(parts_SB[ii] == mypart); |
| 688 |
++ii; |
| 689 |
} |
| 690 |
else { |
| 691 |
parts_SB[ii_ghost] = compute_part(iy, ii_glb); |
| 692 |
local_idx_SB[ii_ghost] = ii_ghost; |
| 693 |
is_ghost_SB[ii_ghost] = true; |
| 694 |
++ii_ghost; |
| 695 |
} |
| 696 |
} |
| 697 |
} |
| 698 |
|
| 699 |
#if DEBUG_OUTPUT_DETAIL |
| 700 |
Log::info() << "[" << mypart << "] : " |
| 701 |
<< "parts_SB = "; |
| 702 |
for (ii = 0; ii < nnodes_SB; ii++) { |
| 703 |
Log::info() << parts_SB[ii] << ","; |
| 704 |
} |
| 705 |
Log::info() << std::endl; |
| 706 |
Log::info() << "[" << mypart << "] : " |
| 707 |
<< "local_idx_SB = "; |
| 708 |
for (ii = 0; ii < nnodes_SB; ii++) { |
| 709 |
Log::info() << local_idx_SB[ii] << ","; |
| 710 |
} |
| 711 |
Log::info() << std::endl; |
| 712 |
Log::info() << "[" << mypart << "] : " |
| 713 |
<< "is_ghost_SB = "; |
| 714 |
for (ii = 0; ii < nnodes_SB; ii++) { |
| 715 |
Log::info() << is_ghost_SB[ii] << ","; |
| 716 |
} |
| 717 |
Log::info() << std::endl; |
| 718 |
#endif |
| 719 |
|
| 720 |
// vectors marking nodes that are necessary for this proc's cells |
| 721 |
std::vector<bool> is_node_SB(nnodes_SB, false); |
| 722 |
|
| 723 |
// determine number of cells and number of nodes |
| 724 |
int nnodes = 0; |
| 725 |
int nquads = 0; |
| 726 |
int npents = 0; |
| 727 |
ii = 0; |
| 728 |
|
| 729 |
int glb2loc_ghost_offset = -nnodes_SB + iy_max + nb_nodes_ + 1; |
| 730 |
auto get_local_id = [this, &parts_sidx, &glb2loc_ghost_offset](gidx_t gidx) { |
| 731 |
return gidx - (gidx < nb_nodes_ ? parts_sidx : glb2loc_ghost_offset); |
| 732 |
}; |
| 733 |
|
| 734 |
for (iy = iy_min; iy <= iy_max; iy++) { |
| 735 |
int nx = nb_lat_nodes(iy); |
| 736 |
for (ix = 0; ix < nx; ix++) { |
| 737 |
if (not is_ghost_SB[ii]) { |
| 738 |
const bool at_pole = (iy == 0 or iy == ny - 1); |
| 739 |
bool not_duplicate_cell = (at_pole ? (ix % 2) : 1); |
| 740 |
if (at_pole and nb_pole_nodes < 8) { |
| 741 |
// nodes at the poles do not own any pentagons |
| 742 |
// if nb_pole_node=1, the node at the poles does not own any quads |
| 743 |
not_duplicate_cell = false; |
| 744 |
} |
| 745 |
if (not at_pole or not_duplicate_cell) { |
| 746 |
if (not is_node_SB[ii]) { |
| 747 |
++nnodes; |
| 748 |
is_node_SB[ii] = true; |
| 749 |
} |
| 750 |
bool is_pentagon = (pole_elements == "pentagons" and (iy == 1 or iy == ny - 2)); |
| 751 |
if (is_pentagon) { |
| 752 |
++npents; |
| 753 |
// mark the pentagon right node |
| 754 |
idx_t iil = get_local_id(pentagon_right_idx(ix, iy, ns)); |
| 755 |
if (not is_node_SB[iil]) { |
| 756 |
++nnodes; |
| 757 |
is_node_SB[iil] = true; |
| 758 |
} |
| 759 |
} |
| 760 |
else { |
| 761 |
++nquads; |
| 762 |
} |
| 763 |
// mark upper corner |
| 764 |
idx_t iil = get_local_id(up_idx(ix, iy, ns)); |
| 765 |
if (not is_node_SB[iil]) { |
| 766 |
++nnodes; |
| 767 |
is_node_SB[iil] = true; |
| 768 |
} |
| 769 |
// mark lower corner |
| 770 |
iil = get_local_id(down_idx(ix, iy, ns)); |
| 771 |
if (not is_node_SB[iil]) { |
| 772 |
++nnodes; |
| 773 |
is_node_SB[iil] = true; |
| 774 |
} |
| 775 |
// mark right corner |
| 776 |
iil = get_local_id(right_idx(ix, iy, ns)); |
| 777 |
if (not is_node_SB[iil]) { |
| 778 |
++nnodes; |
| 779 |
is_node_SB[iil] = true; |
| 780 |
} |
| 781 |
} |
| 782 |
} |
| 783 |
++ii; |
| 784 |
} |
| 785 |
} |
| 786 |
int ncells = nquads + npents; |
| 787 |
ATLAS_ASSERT(ncells > 0); |
| 788 |
|
| 789 |
#if DEBUG_OUTPUT |
| 790 |
Log::info() << "[" << mypart << "] : " |
| 791 |
<< "nnodes = " << nnodes << ", nquads = " << nquads << ", npents = " << npents |
| 792 |
<< ", parts_sidx = " << parts_sidx << std::endl; |
| 793 |
Log::info() << "[" << mypart << "] : " |
| 794 |
<< "iy_min = " << iy_min << ", iy_max = " << iy_max << std::endl; |
| 795 |
#endif |
| 796 |
#if DEBUG_OUTPUT_DETAIL |
| 797 |
Log::info() << "[" << mypart << "] : " |
| 798 |
<< "is_node_SB = "; |
| 799 |
for (int ii = 0; ii < nnodes_SB; ii++) { |
| 800 |
Log::info() << is_node_SB[ii] << ","; |
| 801 |
} |
| 802 |
Log::info() << std::endl; |
| 803 |
#endif |
| 804 |
|
| 805 |
// define nodes and associated properties |
| 806 |
mesh.nodes().resize(nnodes); |
| 807 |
mesh::Nodes& nodes = mesh.nodes(); |
| 808 |
auto xy = array::make_view<double, 2>(nodes.xy()); |
| 809 |
auto lonlat = array::make_view<double, 2>(nodes.lonlat()); |
| 810 |
auto glb_idx = array::make_view<gidx_t, 1>(nodes.global_index()); |
| 811 |
auto remote_idx = array::make_indexview<idx_t, 1>(nodes.remote_index()); |
| 812 |
auto part = array::make_view<int, 1>(nodes.partition()); |
| 813 |
auto ghost = array::make_view<int, 1>(nodes.ghost()); |
| 814 |
auto halo = array::make_view<int, 1>(nodes.halo()); |
| 815 |
auto flags = array::make_view<int, 1>(nodes.flags()); |
| 816 |
|
| 817 |
int quad_begin; |
| 818 |
int pent_begin; |
| 819 |
|
| 820 |
// define cells and associated properties |
| 821 |
mesh.cells().add(mesh::ElementType::create("Quadrilateral"), nquads); |
| 822 |
quad_begin = mesh.cells().elements(0).begin(); |
| 823 |
pent_begin = quad_begin + mesh.cells().elements(0).size(); |
| 824 |
if (pole_elements == "pentagons") { |
| 825 |
mesh.cells().add(mesh::ElementType::create("Pentagon"), npents); |
| 826 |
pent_begin = mesh.cells().elements(1).begin(); |
| 827 |
} |
| 828 |
auto cells_part = array::make_view<int, 1>(mesh.cells().partition()); |
| 829 |
auto cells_glb_idx = array::make_view<gidx_t, 1>(mesh.cells().global_index()); |
| 830 |
auto& node_connectivity = mesh.cells().node_connectivity(); |
| 831 |
|
| 832 |
std::array<idx_t,5> cell_nodes; |
| 833 |
int jquadcell = quad_begin; |
| 834 |
int jpentcell = pent_begin; |
| 835 |
|
| 836 |
int inode_nonghost, inode_ghost; |
| 837 |
|
| 838 |
// loop over nodes and set properties |
| 839 |
inode_nonghost = 0; |
| 840 |
inode_ghost = nnodes_nonghost; // ghost nodes start counting after nonghost nodes |
| 841 |
ii = 0; // index inside SB |
| 842 |
for (iy = iy_min; iy <= iy_max; iy++) { |
| 843 |
int nx = nb_lat_nodes(iy) + 1; |
| 844 |
for (ix = 0; ix < nx; ix++) { |
| 845 |
int iil = idx_xy_to_x(ix, iy, ns); |
| 846 |
iil -= (iil < nb_nodes_ ? parts_sidx : -nnodes_SB + iy_max + nb_nodes_ + 1); |
| 847 |
if (is_node_SB[iil]) { |
| 848 |
// set node counter |
| 849 |
if (is_ghost_SB[iil]) { |
| 850 |
inode = inode_ghost++; |
| 851 |
} |
| 852 |
else { |
| 853 |
inode = inode_nonghost++; |
| 854 |
} |
| 855 |
|
| 856 |
// flags |
| 857 |
Topology::reset(flags(inode)); |
| 858 |
|
| 859 |
glb_idx(inode) = 1 + match_node_idx(idx_xy_to_x(ix, iy, ns), ns); |
| 860 |
|
| 861 |
// grid coordinates |
| 862 |
double _xy[2]; |
| 863 |
double xy1[2], xy2[2]; |
| 864 |
if (iy == 0) { |
| 865 |
_xy[0] = (nb_pole_nodes == 8 ? 45. * ix : (nb_pole_nodes == 4 ? 90. * ix : 180.)); |
| 866 |
_xy[1] = 90.; |
| 867 |
Topology::set(flags(inode), Topology::BC); |
| 868 |
} |
| 869 |
else if (iy == ny - 1) { |
| 870 |
_xy[0] = (nb_pole_nodes == 8 ? 45. * ix : (nb_pole_nodes == 4 ? 90. * ix : 180.)); |
| 871 |
_xy[1] = -90.; |
| 872 |
Topology::set(flags(inode), Topology::BC); |
| 873 |
} |
| 874 |
else if (ix == nx - 1) { |
| 875 |
grid.xy(ix - 1, iy - 1, xy1); |
| 876 |
grid.xy(ix - 2, iy - 1, xy2); |
| 877 |
_xy[0] = 1.5 * xy1[0] - 0.5 * xy2[0]; |
| 878 |
_xy[1] = xy1[1]; |
| 879 |
Topology::set(flags(inode), Topology::BC); |
| 880 |
} |
| 881 |
else if (ix == 0) { |
| 882 |
grid.xy(ix + 1, iy - 1, xy1); |
| 883 |
grid.xy(ix, iy - 1, xy2); |
| 884 |
_xy[0] = 1.5 * xy2[0] - 0.5 * xy1[0]; |
| 885 |
_xy[1] = xy1[1]; |
| 886 |
Topology::set(flags(inode), Topology::BC); |
| 887 |
} |
| 888 |
else { |
| 889 |
grid.xy(ix, iy - 1, xy1); |
| 890 |
grid.xy(ix - 1, iy - 1, xy2); |
| 891 |
_xy[0] = 0.5 * (xy1[0] + xy2[0]); |
| 892 |
_xy[1] = xy1[1]; |
| 893 |
} |
| 894 |
|
| 895 |
if (Topology::check(flags(inode), Topology::BC)) { |
| 896 |
if (iy == 0) { |
| 897 |
Topology::set(flags(inode), Topology::NORTH); |
| 898 |
} |
| 899 |
else if (iy == ny - 1) { |
| 900 |
Topology::set(flags(inode), Topology::SOUTH); |
| 901 |
} |
| 902 |
if (ix == 0) { |
| 903 |
Topology::set(flags(inode), Topology::WEST); |
| 904 |
} |
| 905 |
else if (ix == nx - 1) { |
| 906 |
Topology::set(flags(inode), Topology::EAST | Topology::GHOST); |
| 907 |
ATLAS_ASSERT(is_ghost_SB[iil]); |
| 908 |
} |
| 909 |
} |
| 910 |
|
| 911 |
xy(inode, LON) = _xy[LON]; |
| 912 |
xy(inode, LAT) = _xy[LAT]; |
| 913 |
|
| 914 |
// geographic coordinates by using projection |
| 915 |
grid.projection().xy2lonlat(_xy); |
| 916 |
lonlat(inode, LON) = _xy[LON]; |
| 917 |
lonlat(inode, LAT) = _xy[LAT]; |
| 918 |
|
| 919 |
part(inode) = parts_SB[iil]; |
| 920 |
ghost(inode) = is_ghost_SB[iil]; |
| 921 |
halo(inode) = 0; |
| 922 |
|
| 923 |
if (ghost(inode)) { |
| 924 |
Topology::set(flags(inode), Topology::GHOST); |
| 925 |
remote_idx(inode) = local_idx_SB[iil]; |
| 926 |
} |
| 927 |
else { |
| 928 |
remote_idx(inode) = -1; |
| 929 |
} |
| 930 |
if (Topology::check(flags(inode), Topology::BC | Topology::EAST)) { |
| 931 |
part(inode) = mypart; // To be fixed later |
| 932 |
} |
| 933 |
local_idx_SB[iil] = inode; |
| 934 |
|
| 935 |
#if DEBUG_OUTPUT_DETAIL |
| 936 |
Log::info() << "[" << mypart << "] : " |
| 937 |
<< "New node \tinode=" << inode << "; iil= " << iil << "; ix=" << ix << "; iy=" << iy |
| 938 |
<< "; glon=" << lonlat(inode, 0) << "; glat=" << lonlat(inode, 1) |
| 939 |
<< "; glb_idx=" << glb_idx(inode) << "; loc_idx=" << local_idx_SB[iil] << std::endl; |
| 940 |
#endif |
| 941 |
} |
| 942 |
ii += (ix != nx - 1 ? 1 : 0); |
| 943 |
} |
| 944 |
} |
| 945 |
|
| 946 |
auto get_local_idx_SB = [&local_idx_SB, &get_local_id](gidx_t gidx) { return local_idx_SB[get_local_id(gidx)]; }; |
| 947 |
|
| 948 |
ii = 0; // index inside SB (surrounding belt) |
| 949 |
int jcell_offset = 0; // global index offset due to extra points at the north pole |
| 950 |
gidx_t jcell = 0; // global cell counter |
| 951 |
gidx_t points_in_partition = 0; |
| 952 |
for (iy = iy_min; iy <= iy_max; iy++) { |
| 953 |
int nx = nb_lat_nodes(iy) + 1; |
| 954 |
points_in_partition += (nx - 1); |
| 955 |
for (ix = 0; ix < nx; ix++) { |
| 956 |
const bool at_pole = (iy == 0 or iy == ny - 1); |
| 957 |
int not_duplicate_cell = (at_pole ? ix % 2 : 1); |
| 958 |
if (at_pole and (not not_duplicate_cell or nb_pole_nodes < 8)) { |
| 959 |
// if nb_pole_nodes = 4 : the pole nodes do not own any pentagons |
| 960 |
// if nb_pole_nodes = 1 : the pole nodes do not own any quads |
| 961 |
continue; |
| 962 |
} |
| 963 |
bool pentagon = (iy == 1 or iy == 4 * ns - 1) and pole_elements == "pentagons"; |
| 964 |
int iil = idx_xy_to_x(ix, iy, ns); |
| 965 |
iil -= (iil < nb_nodes_ ? parts_sidx : -nnodes_SB + iy_max + nb_nodes_ + 1); |
| 966 |
if (not is_ghost_SB[iil]) { |
| 967 |
jcell++; // a cell will be added |
| 968 |
// define cell vertices (in local indices) in cell_nodes |
| 969 |
int node_id = 0; |
| 970 |
cell_nodes[node_id++] = get_local_idx_SB(idx_xy_to_x(ix, iy, ns)); |
| 971 |
cell_nodes[node_id++] = get_local_idx_SB(down_idx(ix, iy, ns)); |
| 972 |
bool south_hemisphere = (iy > 2 * ns); |
| 973 |
if (pentagon && not south_hemisphere) { |
| 974 |
cell_nodes[node_id++] = get_local_idx_SB(pentagon_right_idx(ix, iy, ns)); |
| 975 |
} |
| 976 |
cell_nodes[node_id++] = get_local_idx_SB(right_idx(ix, iy, ns)); |
| 977 |
if (pentagon && south_hemisphere) { |
| 978 |
// in the south hemisphere the pentagon point comes in a different clock-wise ordering |
| 979 |
cell_nodes[node_id++] = get_local_idx_SB(pentagon_right_idx(ix, iy, ns)); |
| 980 |
} |
| 981 |
cell_nodes[node_id++] = get_local_idx_SB(up_idx(ix, iy, ns)); |
| 982 |
|
| 983 |
// match global cell indexing for the three healpix versions |
| 984 |
if (nb_pole_nodes == 1) { |
| 985 |
cells_glb_idx(jquadcell) = parts_sidx + points_in_partition - nx + ix + 1; |
| 986 |
} |
| 987 |
else if (nb_pole_nodes == 8) { |
| 988 |
if (iy == 0) { |
| 989 |
cells_glb_idx(jquadcell) = 12 * ns * ns + 1 + ix / 2; |
| 990 |
jcell_offset++; |
| 991 |
} |
| 992 |
else if (iy == ny - 1) { |
| 993 |
cells_glb_idx(jquadcell) = 12 * ns * ns + 5 + ix / 2; |
| 994 |
jcell_offset++; |
| 995 |
} |
| 996 |
else { |
| 997 |
cells_glb_idx(jquadcell) = parts_sidx + iil - 3 - (mypart != 0 ? 4 : jcell_offset); |
| 998 |
} |
| 999 |
} |
| 1000 |
else if (nb_pole_nodes == 4) { |
| 1001 |
if (iy == 0 or iy == ny - 1) { |
| 1002 |
continue; |
| 1003 |
} |
| 1004 |
if (pentagon) { |
| 1005 |
cells_glb_idx(jpentcell) = (mypart != 0 ? 12 * ns * ns - 3 + ix : jcell); |
| 1006 |
jcell_offset++; |
| 1007 |
} |
| 1008 |
else { |
| 1009 |
cells_glb_idx(jquadcell) = parts_sidx + points_in_partition - nx - 6 + ix + (mypart != 0 ? 4 : jcell_offset); |
| 1010 |
} |
| 1011 |
} |
| 1012 |
#if DEBUG_OUTPUT_DETAIL |
| 1013 |
std::cout << "[" << mypart << "] : "; |
| 1014 |
if (pentagon) { |
| 1015 |
std::cout << "New pent: loc-idx " << jpentcell << ", glb-idx " << cells_glb_idx(jpentcell) << ": "; |
| 1016 |
} |
| 1017 |
else { |
| 1018 |
std::cout << "New quad: loc-idx " << jquadcell << ", glb-idx " << cells_glb_idx(jquadcell) << ": "; |
| 1019 |
} |
| 1020 |
std::cout << glb_idx(cell_nodes[0]) << "," << glb_idx(cell_nodes[1]) << "," << glb_idx(cell_nodes[2]) |
| 1021 |
<< "," << glb_idx(cell_nodes[3]); |
| 1022 |
if (pentagon) { |
| 1023 |
std::cout << "," << glb_idx(cell_nodes[4]); |
| 1024 |
} |
| 1025 |
std::cout << std::endl; |
| 1026 |
#endif |
| 1027 |
// add cell to the node connectivity table |
| 1028 |
if (pentagon) { |
| 1029 |
if (south_hemisphere) { |
| 1030 |
// Rotate by 2 so that output in 3d shows healpix quad when taking the first 4 nodes |
| 1031 |
// |
| 1032 |
// 5 3 |
| 1033 |
// .´ `. .´ `. |
| 1034 |
// 1 4 ----rotate-by-2---> 4 2 |
| 1035 |
// | | | | |
| 1036 |
// 2------3 --- south pole --- 5------1 |
| 1037 |
// |
| 1038 |
std::rotate(cell_nodes.begin(),cell_nodes.begin()+2,cell_nodes.end()); |
| 1039 |
} |
| 1040 |
cells_part(jpentcell) = mypart; |
| 1041 |
node_connectivity.set(jpentcell, cell_nodes.data()); |
| 1042 |
++jpentcell; |
| 1043 |
} |
| 1044 |
else { |
| 1045 |
cells_part(jquadcell) = mypart; |
| 1046 |
node_connectivity.set(jquadcell, cell_nodes.data()); |
| 1047 |
++jquadcell; |
| 1048 |
} |
| 1049 |
} |
| 1050 |
ii += (ix != nx - 1 ? 1 : 0); |
| 1051 |
} |
| 1052 |
} |
| 1053 |
|
| 1054 |
#if DEBUG_OUTPUT_DETAIL |
| 1055 |
// list nodes |
| 1056 |
Log::info() << "Listing nodes ..."; |
| 1057 |
for (inode = 0; inode < nnodes; inode++) { |
| 1058 |
std::cout << "[" << mypart << "] : " |
| 1059 |
<< " node " << inode << ": ghost = " << ghost(inode) << ", glb_idx = " << glb_idx(inode) |
| 1060 |
<< ", part = " << part(inode) << ", lon = " << lonlat(inode, 0) << ", lat = " << lonlat(inode, 1) |
| 1061 |
<< ", remote_idx = " << remote_idx(inode) << std::endl; |
| 1062 |
} |
| 1063 |
|
| 1064 |
for (gidx_t jcell = quad_begin; jcell < nquads; jcell++) { |
| 1065 |
std::cout << "[" << mypart << "] : " |
| 1066 |
<< " cell " << jcell << ", glb-idx " << cells_glb_idx(jcell) << ": " |
| 1067 |
<< glb_idx(node_connectivity(jcell, 0)) << "," << glb_idx(node_connectivity(jcell, 1)) << "," |
| 1068 |
<< glb_idx(node_connectivity(jcell, 2)) << "," << glb_idx(node_connectivity(jcell, 3)) << std::endl; |
| 1069 |
} |
| 1070 |
for (gidx_t jcell = pent_begin; jcell < nquads; jcell++) { |
| 1071 |
std::cout << "[" << mypart << "] : " |
| 1072 |
<< " cell " << jcell << ", glb-idx " << cells_glb_idx(jcell) << ": " |
| 1073 |
<< glb_idx(node_connectivity(jcell, 0)) << "," << glb_idx(node_connectivity(jcell, 1)) << "," |
| 1074 |
<< glb_idx(node_connectivity(jcell, 2)) << "," << glb_idx(node_connectivity(jcell, 3)) << std::endl; |
| 1075 |
} |
| 1076 |
#endif |
| 1077 |
|
| 1078 |
mesh.metadata().set("nb_parts",options.getInt("nb_parts")); |
| 1079 |
mesh.metadata().set("part",options.getInt("part")); |
| 1080 |
mesh.metadata().set("mpi_comm",options.getString("mpi_comm")); |
| 1081 |
mesh.metadata().set<size_t>("nb_nodes_including_halo[0]", nodes.size()); |
| 1082 |
nodes.metadata().set<size_t>("NbRealPts", size_t(nnodes)); |
| 1083 |
nodes.metadata().set<size_t>("NbVirtualPts", size_t(0)); |
| 1084 |
nodes.global_index().metadata().set("human_readable", true); |
| 1085 |
nodes.global_index().metadata().set("min", 1); |
| 1086 |
nodes.global_index().metadata().set("max", nb_nodes_ + grid.ny() + 2); |
| 1087 |
mesh.cells().global_index().metadata().set("human_readable", true); |
| 1088 |
mesh.cells().global_index().metadata().set("min", 1); |
| 1089 |
mesh.cells().global_index().metadata().set("max", nb_points_); |
| 1090 |
|
| 1091 |
//generateGlobalElementNumbering(mesh); |
| 1092 |
} // generate_mesh |
| 1093 |
|
| 1094 |
namespace { |
| 1095 |
static MeshGeneratorBuilder<HealpixMeshGenerator> __HealpixMeshGenerator(HealpixMeshGenerator::static_type()); |
| 1096 |
} |
| 1097 |
|
| 1098 |
} // namespace meshgenerator |
| 1099 |
} // namespace atlas |