Debian Codegraph @

src/atlas/meshgenerator/detail/CubedSphereMeshGenerator.cc

raw
1 /*
2 * (C) Crown Copyright 2021 Met Office
3 *
4 * This software is licensed under the terms of the Apache Licence Version 2.0
5 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
6 */
7
8 #include <algorithm>
9 #include <cmath>
10 #include <limits>
11 #include <numeric>
12 #include <string>
13 #include <vector>
14
15 #include "eckit/utils/Hash.h"
16
17 #include "atlas/field/Field.h"
18 #include "atlas/grid/CubedSphereGrid.h"
19 #include "atlas/grid/Distribution.h"
20 #include "atlas/grid/Iterator.h"
21 #include "atlas/grid/Partitioner.h"
22 #include "atlas/grid/Tiles.h"
23 #include "atlas/library/config.h"
24 #include "atlas/mesh/Connectivity.h"
25 #include "atlas/mesh/ElementType.h"
26 #include "atlas/mesh/Elements.h"
27 #include "atlas/mesh/HybridElements.h"
28 #include "atlas/mesh/Mesh.h"
29 #include "atlas/mesh/Nodes.h"
30 #include "atlas/meshgenerator/detail/CubedSphereMeshGenerator.h"
31 #include "atlas/meshgenerator/detail/MeshGeneratorFactory.h"
32 #include "atlas/meshgenerator/detail/cubedsphere/CubedSphereUtility.h"
33 #include "atlas/parallel/mpi/mpi.h"
34 #include "atlas/projection/detail/CubedSphereProjectionBase.h"
35 #include "atlas/runtime/Exception.h"
36 #include "atlas/runtime/Log.h"
37 #include "atlas/util/Constants.h"
38 #include "atlas/util/CoordinateEnums.h"
39
40 #define DEBUG_OUTPUT 0
41 #define DEBUG_OUTPUT_DETAIL 0
42
43 namespace atlas {
44 namespace meshgenerator {
45
46 // -----------------------------------------------------------------------------
47
48 CubedSphereMeshGenerator::CubedSphereMeshGenerator(const eckit::Parametrisation& p) {
49 // Get mpi_comm
50 std::string mpi_comm = mpi::comm().name();
51 p.get("mpi_comm", mpi_comm);
52 options.set("mpi_comm", mpi_comm);
53
54 configure_defaults();
55
56 // Get number of partitions.
57 int nb_parts;
58 if (p.get("nb_parts", nb_parts)) {
59 options.set("nb_parts", nb_parts);
60 }
61
62 // Get this partition.
63 int part;
64 if (p.get("part", part)) {
65 options.set("part", part);
66 }
67
68 // Get halo size.
69 int halo;
70 if (p.get("halo", halo)) {
71 options.set("halo", halo);
72 }
73
74 // Get partitioner.
75 std::string partitioner;
76 if (p.get("partitioner", partitioner) && partitioner.size()) {
77 options.set("partitioner", partitioner);
78 }
79 }
80
81 // -----------------------------------------------------------------------------
82
83
84 void CubedSphereMeshGenerator::configure_defaults() {
85 auto& comm = mpi::comm(options.getString("mpi_comm"));
86
87 // This option sets number of partitions.
88 options.set("nb_parts", comm.size());
89
90 // This option sets the part that will be generated.
91 options.set("part", comm.rank());
92
93 // This options sets the number of halo elements around each partition.
94 options.set("halo", 0);
95
96 // This options sets the default partitioner.
97 options.set<std::string>("partitioner", "cubedsphere");
98 }
99
100 // -----------------------------------------------------------------------------
101
102 void CubedSphereMeshGenerator::generate(const Grid& grid, Mesh& mesh) const {
103 // Get partitioner type and number of partitions from config.
104 const idx_t nParts = static_cast<idx_t>(options.get<size_t>("nb_parts"));
105 const std::string partType = options.get<std::string>("partitioner");
106
107 auto partConfig = util::Config{};
108 partConfig.set("type", partType);
109 partConfig.set("partitions", nParts);
110
111 // Use lonlat instead of xy for non cubedsphere partitioner.
112 if (partType != "cubedsphere") {
113 partConfig.set("coordinates", "lonlat");
114 }
115
116 // Set distribution.
117 mpi::Scope mpi_scope(options.getString("mpi_comm"));
118 const auto partitioner = grid::Partitioner(partConfig);
119 const auto distribution = grid::Distribution(grid, partitioner);
120
121 generate(grid, distribution, mesh);
122 }
123
124 // -----------------------------------------------------------------------------
125
126 void CubedSphereMeshGenerator::generate(const Grid& grid, const grid::Distribution& distribution, Mesh& mesh) const {
127 // Check for correct grid and need for mesh
128 ATLAS_ASSERT(!mesh.generated());
129
130 // Cast grid to cubed sphere grid.
131 const auto csGrid = CubedSphereGrid(grid);
132
133 // Check for successful cast.
134 if (!csGrid) {
135 throw_Exception(
136 "CubedSphereMeshGenerator can only work "
137 "with a cubedsphere grid.",
138 Here());
139 }
140
141 // Check for correct grid stagger.
142 if (csGrid.stagger() != "C") {
143 throw_Exception(
144 "CubedSphereMeshGenerator will only work with a "
145 "cell-centroid grid.",
146 Here());
147 }
148
149 // Check for sensible halo size.
150 if (options.get<idx_t>("halo") > csGrid.N()) {
151 throw_Exception("Halo size " + std::to_string(options.get<idx_t>("halo")) +
152 " "
153 "is larger than grid size " +
154 std::to_string(csGrid.N()) + ".",
155 Here());
156 }
157
158 // Clone some grid properties.
159 setGrid(mesh, csGrid, distribution);
160 mesh.metadata().set("mpi_comm",options.getString("mpi_comm"));
161
162 generate_mesh(csGrid, distribution, mesh);
163 }
164
165 // -----------------------------------------------------------------------------
166
167 void CubedSphereMeshGenerator::generate_mesh(const CubedSphereGrid& csGrid, const grid::Distribution& distribution,
168 Mesh& mesh) const {
169 ATLAS_TRACE("CubedSphereMeshGenerator::generate");
170
171 // ---------------------------------------------------------------------------
172 // CUBED SPHERE MESH GENERATOR
173 // ---------------------------------------------------------------------------
174 //
175 // Mesh generator creates a cubed sphere mesh by generating individual meshes
176 // for each cubed sphere tile and then working out the correspondence between
177 // overlapping nodes and cells.
178 //
179 // Meshgenerator places cell at each grid point on this partition. Halo
180 // cells are added to the mesh if options.get("halo") > 0. The halo cells may
181 // either be interior to the tile, || exterior. Interior halo cells share
182 // their xy coordinate and global ID with the corresponding cells on other
183 // partitions. Exterior halo cells have a unique global ID and their xy
184 // coordinates are extrapolated from the interior tile. The global IDs of non-
185 // halo cells match the global IDs of the cell-centre grid points.
186 //
187 // Nodes are added around cells. The nodes around interior cells are assigned
188 // an owner by the following rules:
189 // * node (i > 0, j > 0) is owned by cell (i - 1, j - 1)
190 // * node (i = 0, j > 0) is owned by cell (0 , j - 1)
191 // * node (i > 0, j = 0) is owned by cell (i - 1, 0 )
192 // * node (i = 0, j = 0) is owned by cell (0 , 0 )
193 //
194 // The partition of the owning cell determines the partition of the node.
195 // Ghost nodes are added to the mesh to complete cells at partition
196 // boundaries, cells exterior to the tiles, or on tile edges which do not
197 // own nodes.
198 //
199 // There are several stages to the mesh generator:
200 // 1. Preamble.
201 // 2. Define global cell distribution.
202 // 3. Define global node distribution.
203 // 4. Locate local cells.
204 // 5. Locate local nodes
205 // 6. Assign nodes to mesh.
206 // 7. Assign cells to mesh.
207 // 8. Finalise.
208
209 // ---------------------------------------------------------------------------
210 // 1. PREAMBLE
211 // Setup some general parameters of the mesh, as well as define useful
212 // lambda functions and structs.
213 // ---------------------------------------------------------------------------
214
215 using Topology = atlas::mesh::Nodes::Topology;
216 using atlas::array::make_datatype;
217 using atlas::array::make_shape;
218
219 using namespace detail::cubedsphere;
220
221 struct GlobalElem;
222 struct LocalElem;
223
224 // Define bad index values.
225 constexpr idx_t undefinedIdx = -1;
226 constexpr idx_t undefinedGlobalIdx = -1;
227
228 // Define cell/node type.
229 enum class ElemType
230 {
231 UNDEFINED, // Not set. Writing this type to mesh is an error.
232 OWNER, // Owner element on this partition.
233 HALO // Ghost or halo element.
234 };
235
236 // Struct to store the information of globals cells/nodes.
237 // These must be generated for all cells/nodes on all partitions. Data is
238 // stored in (t, j, i) row-major order.
239 struct GlobalElem {
240 LocalElem* localPtr{}; // Pointer to local element.
241 gidx_t globalIdx{undefinedGlobalIdx}; // Global index.
242 idx_t remoteIdx{undefinedIdx}; // Remote index.
243 int part{undefinedIdx}; // Partition.
244 };
245
246 // Struct to store additional information of local cells/nodes.
247 // These are only generated for nodes on this partition. Data is stored as a
248 // list which can be sorted.
249 struct LocalElem {
250 GlobalElem* globalPtr{}; // Pointer to global element.
251 ElemType type{ElemType::UNDEFINED}; // Cell/node Type.
252 int halo{undefinedIdx}; // Halo level.
253 idx_t t{}; // t, i and j.
254 idx_t i{};
255 idx_t j{};
256 };
257
258 // Define ij bounding box for each face (this partition only).
259 struct BoundingBox {
260 idx_t iBegin{std::numeric_limits<idx_t>::max()};
261 idx_t iEnd{std::numeric_limits<idx_t>::min()};
262 idx_t jBegin{std::numeric_limits<idx_t>::max()};
263 idx_t jEnd{std::numeric_limits<idx_t>::min()};
264 };
265
266 // Get dimensions of grid
267 const idx_t N = csGrid.N();
268
269 // Get size of halo.
270 int nHalo = 0;
271 options.get("halo", nHalo);
272
273 // Unique non-halo nodes and cells.
274 const idx_t nNodesUnique = 6 * N * N + 2;
275 const idx_t nCellsUnique = 6 * N * N;
276
277 // Total array sizes (including invalid corner ijs).
278 const idx_t nNodesArray = 6 * (N + 2 * nHalo + 1) * (N + 2 * nHalo + 1);
279 const idx_t nCellsArray = 6 * (N + 2 * nHalo) * (N + 2 * nHalo);
280
281 // Total number of possible cells and nodes (including halos).
282 const idx_t nNodesTotal = nNodesArray - 6 * 4 * nHalo * nHalo;
283 const idx_t nCellsTotal = nCellsArray - 6 * 4 * nHalo * nHalo;
284
285 // Projection and jacobian.
286 const auto& csProjection = csGrid.cubedSphereProjection();
287 const auto jacobian = NeighbourJacobian(csGrid);
288
289 // Get partition information.
290 const int nParts = options.get<int>("nb_parts");
291 const int thisPart = options.get<int>("part");
292
293 // Define an index counter.
294 const auto idxSum = [](const std::vector<idx_t>& idxCounts) -> idx_t {
295 return std::accumulate(idxCounts.begin(), idxCounts.end(), 0);
296 };
297
298 // Helper functions to get node vector idx from (i, j, t).
299 const auto getNodeIdx = [&](idx_t i, idx_t j, idx_t t) -> size_t {
300 const idx_t rowSize = (N + 2 * nHalo + 1);
301 const idx_t tileSize = rowSize * rowSize;
302
303 return static_cast<size_t>(t * tileSize + (j + nHalo) * rowSize + i + nHalo);
304 };
305
306 // Helper functions to get cell vector idx from (i, j, t).
307 const auto getCellIdx = [&](idx_t i, idx_t j, idx_t t) -> size_t {
308 const idx_t rowSize = (N + 2 * nHalo);
309 const idx_t tileSize = rowSize * rowSize;
310
311 return static_cast<size_t>(t * tileSize + (j + nHalo) * rowSize + i + nHalo);
312 };
313
314 // Return true for nodes interior to tile (excluding edge).
315 const auto interiorNode = [&](idx_t i, idx_t j) -> bool { return i > 0 && i < N && j > 0 && j < N; };
316
317 // return true for nodes exterior to tile (excluding edge).
318 const auto exteriorNode = [&](idx_t i, idx_t j) -> bool { return i < 0 || i > N || j < 0 || j > N; };
319
320 // Return true for cells interior to tile.
321 const auto interiorCell = [&](idx_t i, idx_t j) -> bool { return i >= 0 && i < N && j >= 0 && j < N; };
322
323 // Return true for impossible node (i, j) combinations.
324 const auto invalidNode = [&](idx_t i, idx_t j) -> bool {
325 const bool inCorner = (i < 0 && j < 0) || // Bottom-left corner.
326 (i > N && j < 0) || // Bottom-right corner.
327 (i > N && j > N) || // Top-right corner.
328 (i < 0 && j > N); // Top-left corner.
329
330 if (inCorner) {
331 return true;
332 }
333
334 const bool outOfBounds = i < -nHalo || i > N + nHalo || j < -nHalo || j > N + nHalo;
335
336 if (outOfBounds) {
337 return true;
338 }
339
340 return false;
341 };
342
343 // Return true for impossible cell (i, j) combinations.
344 const auto invalidCell = [&](idx_t i, idx_t j) -> bool {
345 const bool inCorner = (i < 0 && j < 0) || // Bottom-left corner.
346 (i > N - 1 && j < 0) || // Bottom-right corner.
347 (i > N - 1 && j > N - 1) || // Top-right corner.
348 (i < 0 && j > N - 1); // Top-left corner.
349
350 if (inCorner) {
351 return true;
352 }
353
354 const bool outOfBounds = i < -nHalo || i > N + nHalo - 1 || j < -nHalo || j > N + nHalo - 1;
355
356 if (outOfBounds) {
357 return true;
358 }
359
360 return false;
361 };
362
363 // Return the (i, j) of cell that owns node (i, j).
364 // It's possible that this may need to be user-defined in the future.
365 const auto nodeOwnerCell = [](idx_t iNode, idx_t jNode) -> std::pair<idx_t, idx_t> {
366 return std::make_pair(iNode > 0 ? iNode - 1 : iNode, jNode > 0 ? jNode - 1 : jNode);
367 };
368
369 // ---------------------------------------------------------------------------
370 // 2. GLOBAL CELL DISTRIBUTION
371 // Need to construct a lightweight global mesh. This will help us pair up
372 // local and remote indices on different partitions.
373 // ---------------------------------------------------------------------------
374
375 // Make vector of all possible cells.
376 auto globalCells = std::vector<GlobalElem>(static_cast<size_t>(nCellsArray));
377
378 // Initialise bounding box.
379 auto cellBounds = std::vector<BoundingBox>(6);
380
381 // Set tij grid iterator.
382 auto tijIt = csGrid.tij().begin();
383
384 for (gidx_t gridIdx = 0; gridIdx < csGrid.size(); ++gridIdx) {
385 // It's more than likely that the grid order follows the same (t, j, i) row-
386 // major order of the mesh. However, we shouldn't assume this is true.
387
388 // Get cell index.
389 const idx_t t = (*tijIt).t();
390 const idx_t i = (*tijIt).i();
391 const idx_t j = (*tijIt).j();
392 const size_t cellIdx = getCellIdx(i, j, t);
393 GlobalElem& globalCell = globalCells[cellIdx];
394
395 // Set global index.
396 globalCell.globalIdx = gridIdx + 1;
397
398 // Set partition.
399 globalCell.part = distribution.partition(gridIdx);
400
401 if (globalCell.part == static_cast<idx_t>(thisPart)) {
402 // Keep track of local (t, j, i) bounds.
403 BoundingBox& bounds = cellBounds[static_cast<size_t>(t)];
404 bounds.iBegin = std::min(bounds.iBegin, i - nHalo);
405 bounds.iEnd = std::max(bounds.iEnd, i + nHalo + 1);
406 bounds.jBegin = std::min(bounds.jBegin, j - nHalo);
407 bounds.jEnd = std::max(bounds.jEnd, j + nHalo + 1);
408 }
409 // Increment tij.
410 ++tijIt;
411 }
412
413 // Set counters for cell local indices.
414 auto cellLocalIdxCount = std::vector<idx_t>(nParts, 0);
415
416 // Give possible edge-halo cells a unique global ID.
417 gidx_t cellGlobalIdxCount = nCellsUnique + 1;
418
419 for (idx_t t = 0; t < 6; ++t) {
420 for (idx_t j = -nHalo; j < N + nHalo; ++j) {
421 for (idx_t i = -nHalo; i < N + nHalo; ++i) {
422 // Skip invalid cell.
423 if (invalidCell(i, j)) {
424 continue;
425 }
426
427 // Get cell.
428 GlobalElem& globalCell = globalCells[getCellIdx(i, j, t)];
429
430 // Set cell remote index if interior cell.
431 if (interiorCell(i, j)) {
432 // Set remote index.
433 globalCell.remoteIdx = cellLocalIdxCount[static_cast<size_t>(globalCell.part)]++;
434 }
435 else {
436 // Set global index.
437 globalCell.globalIdx = cellGlobalIdxCount++;
438 // Leave remote index and part undefined.
439 }
440 }
441 }
442 }
443
444 ATLAS_ASSERT(idxSum(cellLocalIdxCount) == nCellsUnique);
445 ATLAS_ASSERT(cellGlobalIdxCount == nCellsTotal + 1);
446
447 // ---------------------------------------------------------------------------
448 // 3. GLOBAL NODE DISTRIBUTION
449 // Construct a lightweight global distribution of nodes. Again, this will
450 // help us match up local and remote indices accross partitions.
451 // ---------------------------------------------------------------------------
452
453 // Make list of all nodes.
454 auto globalNodes = std::vector<GlobalElem>(static_cast<size_t>(nNodesArray));
455
456 // Set counters for local node indices.
457 auto nodeLocalIdxCount = std::vector<idx_t>(nParts, 0);
458
459 // Set counter for global indices.
460 gidx_t nodeGlobalOwnedIdxCount = 1;
461 idx_t nodeGlobalGhostIdxCount = nNodesUnique + 1;
462
463 for (idx_t t = 0; t < 6; ++t) {
464 for (idx_t j = -nHalo; j < N + nHalo + 1; ++j) {
465 for (idx_t i = -nHalo; i < N + nHalo + 1; ++i) {
466 // Skip if not a valid node.
467 if (invalidNode(i, j)) {
468 continue;
469 }
470
471 // Get this node.
472 GlobalElem& globalNode = globalNodes[getNodeIdx(i, j, t)];
473
474 // Get owner cell.
475 idx_t iCell, jCell;
476 std::tie(iCell, jCell) = nodeOwnerCell(i, j);
477 const GlobalElem& ownerCell = globalCells[getCellIdx(iCell, jCell, t)];
478
479 if (interiorNode(i, j)) {
480 // Node is definitely an owner.
481 globalNode.globalIdx = nodeGlobalOwnedIdxCount++;
482 globalNode.part = ownerCell.part;
483 globalNode.remoteIdx = nodeLocalIdxCount[static_cast<size_t>(globalNode.part)]++;
484 }
485 else if (exteriorNode(i, j)) {
486 // Node is definitely a ghost.
487 globalNode.globalIdx = nodeGlobalGhostIdxCount++;
488 // Leave remote index and partition undefined.
489 }
490 else {
491 // We're not sure (i.e., node is on a tile edge).
492
493 // Check that xy is on this tile.
494 PointXY xy = jacobian.xy(PointIJ(i, j), t);
495 xy = jacobian.snapToEdge(xy, t);
496
497 // This will only determine if tGlobal does not match t.
498 // This is cheaper than determining the correct tGlobal.
499 idx_t tGlobal = csProjection.getCubedSphereTiles().indexFromXY(xy.data());
500
501 if (tGlobal == t) {
502 // Node is an owner.
503 globalNode.globalIdx = nodeGlobalOwnedIdxCount++;
504 globalNode.part = ownerCell.part;
505 globalNode.remoteIdx = nodeLocalIdxCount[static_cast<size_t>(globalNode.part)]++;
506 }
507 else {
508 // Node is a ghost.
509 globalNode.globalIdx = nodeGlobalGhostIdxCount++;
510 }
511 } // Finished with this node.
512 }
513 } // Finished with all nodes on tile.
514 } // Finished with all tiles.
515
516 ATLAS_ASSERT(nodeGlobalOwnedIdxCount == nNodesUnique + 1);
517 ATLAS_ASSERT(nodeGlobalGhostIdxCount == nNodesTotal + 1);
518 ATLAS_ASSERT(idxSum(nodeLocalIdxCount) == nNodesUnique);
519
520 // ---------------------------------------------------------------------------
521 // 4. LOCATE LOCAL CELLS.
522 // Now that we know where all the nodes and cells are, we can make a list
523 // of local cells to add the the mesh. "Owner" cells correspond to grid
524 // points on this parition. "Halo" cells are at most nHalo grid points
525 // away from an owner cell.
526 // ---------------------------------------------------------------------------
527
528 // Make vector of local cells.
529 auto localCells = std::vector<LocalElem>{};
530
531 // Loop over all possible local cells.
532 for (idx_t t = 0; t < 6; ++t) {
533 // Limit range to bounds recorded earlier.
534 const BoundingBox& bounds = cellBounds[static_cast<size_t>(t)];
535
536 for (idx_t j = bounds.jBegin; j < bounds.jEnd; ++j) {
537 for (idx_t i = bounds.iBegin; i < bounds.iEnd; ++i) {
538 if (invalidCell(i, j)) {
539 continue;
540 }
541
542 // Get cell
543 GlobalElem& globalCell = globalCells[getCellIdx(i, j, t)];
544
545 // Check if cell is an owner.
546 if (globalCell.part == static_cast<idx_t>(thisPart)) {
547 // Cell is an owner.
548 localCells.emplace_back();
549 LocalElem& localCell = localCells.back();
550
551 localCell.type = ElemType::OWNER;
552 localCell.halo = 0;
553 localCell.t = t;
554 localCell.i = i;
555 localCell.j = j;
556 localCell.globalPtr = &globalCell;
557 }
558 else {
559 // Cell is halo if there are nearby owners.
560 bool ownerFound = false;
561 idx_t halo = nHalo;
562 for (idx_t jHalo = j - nHalo; jHalo < j + nHalo + 1; ++jHalo) {
563 for (idx_t iHalo = i - nHalo; iHalo < i + nHalo + 1; ++iHalo) {
564 if (invalidCell(iHalo, jHalo) || (iHalo == i && jHalo == j)) {
565 continue;
566 }
567
568 // Is there a nearby owner cell?
569 const GlobalElem& ownerCell = globalCells[getCellIdx(iHalo, jHalo, t)];
570
571 const bool isOwner = ownerCell.part == static_cast<idx_t>(thisPart);
572
573 ownerFound = ownerFound || isOwner;
574
575 if (isOwner) {
576 // Determine halo level from cell distance (l-infinity norm).
577 const idx_t dist = std::max(std::abs(iHalo - i), std::abs(jHalo - j));
578
579 halo = std::min(dist, halo);
580 }
581 }
582 }
583
584 if (ownerFound) {
585 // Cell is a halo.
586 localCells.emplace_back();
587 LocalElem& localCell = localCells.back();
588
589 localCell.type = ElemType::HALO;
590 localCell.halo = halo;
591 localCell.t = t;
592 localCell.i = i;
593 localCell.j = j;
594 localCell.globalPtr = &globalCell;
595
596 } // Finished with halo search.
597 } // Finished with cell.
598 }
599 } // Finished with all cells on tile.
600 } // Finished with all tiles.
601
602 // Partition by cell type.
603 auto haloBeginIt =
604 std::stable_partition(localCells.begin(), localCells.end(),
605 [](const LocalElem& localCell) -> bool { return localCell.type == ElemType::OWNER; });
606
607 // Sort by halo.
608 std::stable_sort(haloBeginIt, localCells.end(),
609 [](const LocalElem& cellA, const LocalElem& cellB) -> bool { return cellA.halo < cellB.halo; });
610
611 // Point global cell to local cell. This is needed to determine node halos.
612 // Need to determine remote index and partition if we haven't done so already.
613 for (LocalElem& localCell : localCells) {
614 localCell.globalPtr->localPtr = &localCell;
615
616 if (localCell.globalPtr->remoteIdx == undefinedIdx) {
617 const PointTIJ tijGlobal =
618 jacobian.ijLocalToGlobal(PointIJ(localCell.i + 0.5, localCell.j + 0.5), localCell.t);
619
620 const idx_t& t = tijGlobal.t();
621 const idx_t& i = tijGlobal.ij().iCell();
622 const idx_t& j = tijGlobal.ij().jCell();
623
624 const GlobalElem& ownerCell = globalCells[getCellIdx(i, j, t)];
625
626 localCell.globalPtr->remoteIdx = ownerCell.remoteIdx;
627 localCell.globalPtr->part = ownerCell.part;
628 }
629 }
630
631 // ---------------------------------------------------------------------------
632 // 5. LOCATE LOCAL NODES.
633 // We can now locate the nodes surrounding the owner and halo cells on the
634 // mesh.
635 // ---------------------------------------------------------------------------
636
637 // Make vector of local nodes.
638 auto localNodes = std::vector<LocalElem>{};
639
640 // Loop over all possible local nodes.
641 for (idx_t t = 0; t < 6; ++t) {
642 // Limit range to bounds recorded earlier.
643 const BoundingBox& bounds = cellBounds[static_cast<size_t>(t)];
644
645 for (idx_t j = bounds.jBegin; j < bounds.jEnd + 1; ++j) {
646 for (idx_t i = bounds.iBegin; i < bounds.iEnd + 1; ++i) {
647 if (invalidNode(i, j)) {
648 continue;
649 }
650
651 // Get node.
652 GlobalElem& globalNode = globalNodes[getNodeIdx(i, j, t)];
653
654 // Check if node is an owner.
655 if (globalNode.part == static_cast<idx_t>(thisPart)) {
656 // Node is an owner.
657 localNodes.emplace_back();
658 LocalElem& localNode = localNodes.back();
659
660 localNode.type = ElemType::OWNER;
661 localNode.halo = 0;
662 localNode.i = i;
663 localNode.j = j;
664 localNode.t = t;
665 localNode.globalPtr = &globalNode;
666 }
667 else {
668 // Node is possibly a ghost or halo.
669
670 // Search four neighbouring cells of node.
671 bool isGhost = false;
672 idx_t halo = nHalo;
673
674 // A double-loop with no more than four iterations!
675 for (idx_t iCell = i - 1; iCell < i + 1; ++iCell) {
676 for (idx_t jCell = j - 1; jCell < j + 1; ++jCell) {
677 if (invalidCell(iCell, jCell)) {
678 continue;
679 }
680
681 const GlobalElem& globalCell = globalCells[getCellIdx(iCell, jCell, t)];
682
683 // Get halo information from cell.
684 // By this point, any global cell on this PE will have a
685 // non-null local cell pointer.
686 if (globalCell.localPtr) {
687 isGhost = true;
688 halo = std::min<idx_t>(halo, globalCell.localPtr->halo);
689 }
690 }
691 }
692
693 if (isGhost) {
694 // Node is an ghost/halo.
695 localNodes.emplace_back();
696 LocalElem& localNode = localNodes.back();
697
698 localNode.type = ElemType::HALO;
699 localNode.halo = halo;
700 localNode.t = t;
701 localNode.i = i;
702 localNode.j = j;
703 localNode.globalPtr = &globalNode;
704 }
705
706 } // Finished with this node.
707 }
708 } // Finished with all nodes on tile.
709 } // Finished with all tiles.
710
711 // Partition by node type.
712 auto ghostBeginIt =
713 std::stable_partition(localNodes.begin(), localNodes.end(),
714 [](const LocalElem& localNode) -> bool { return localNode.type == ElemType::OWNER; });
715
716 // Sort by halo.
717 std::stable_sort(ghostBeginIt, localNodes.end(),
718 [](const LocalElem& nodeA, const LocalElem& nodeB) -> bool { return nodeA.halo < nodeB.halo; });
719
720 // Associate global nodes with local nodes. Allows us to determine node
721 // local indices around a cell.
722 // Determine partition and remote index if we haven't done so already.
723 for (LocalElem& localNode : localNodes) {
724 localNode.globalPtr->localPtr = &localNode;
725
726 if (localNode.globalPtr->remoteIdx == undefinedIdx) {
727 const PointTIJ tijGlobal = jacobian.ijLocalToGlobal(PointIJ(localNode.i, localNode.j), localNode.t);
728
729 const idx_t& t = tijGlobal.t();
730 const idx_t& i = tijGlobal.ij().iNode();
731 const idx_t& j = tijGlobal.ij().jNode();
732
733 const GlobalElem& ownerNode = globalNodes[getNodeIdx(i, j, t)];
734
735 localNode.globalPtr->remoteIdx = ownerNode.remoteIdx;
736 localNode.globalPtr->part = ownerNode.part;
737 }
738 }
739
740 // ---------------------------------------------------------------------------
741 // 6. ASSIGN NODES TO MESH
742 // In addition to the usual fields, we will add t, i and j to mesh.nodes().
743 // ---------------------------------------------------------------------------
744
745 // Resize nodes.
746 auto& nodes = mesh.nodes();
747 nodes.resize(static_cast<idx_t>(localNodes.size()));
748
749 // Add extra field.
750 Field tijField = nodes.add(Field("tij", make_datatype<idx_t>(), make_shape(nodes.size(), 3)));
751 tijField.set_variables(3);
752
753 // Get field views
754 auto nodesGlobalIdx = array::make_view<gidx_t, 1>(nodes.global_index());
755 auto nodesRemoteIdx = array::make_indexview<idx_t, 1>(nodes.remote_index());
756 auto nodesXy = array::make_view<double, 2>(nodes.xy());
757 auto nodesLonLat = array::make_view<double, 2>(nodes.lonlat());
758 auto nodesPart = array::make_view<int, 1>(nodes.partition());
759 auto nodesGhost = array::make_view<int, 1>(nodes.ghost());
760 auto nodesHalo = array::make_view<int, 1>(nodes.halo());
761 auto nodesFlags = array::make_view<int, 1>(nodes.flags());
762 auto nodesTij = array::make_view<idx_t, 2>(tijField);
763
764 // Set fields.
765 idx_t nodeLocalIdx = 0;
766 for (const LocalElem& localNode : localNodes) {
767 // Set global index.
768 nodesGlobalIdx(nodeLocalIdx) = localNode.globalPtr->globalIdx;
769
770 // Set node remote index.
771 nodesRemoteIdx(nodeLocalIdx) = localNode.globalPtr->remoteIdx;
772
773 // Set node partition.
774 nodesPart(nodeLocalIdx) = localNode.globalPtr->part;
775
776 // Set xy.
777 const PointXY xyLocal = jacobian.xy(PointIJ(localNode.i, localNode.j), localNode.t);
778 const PointXY xyGlobal =
779 interiorNode(localNode.i, localNode.j) ? xyLocal : jacobian.xyLocalToGlobal(xyLocal, localNode.t).xy();
780
781 nodesXy(nodeLocalIdx, XX) = xyLocal.x();
782 nodesXy(nodeLocalIdx, YY) = xyLocal.y();
783
784 // Set lon-lat.
785 const PointLonLat lonLat = csProjection.lonlat(xyGlobal);
786 nodesLonLat(nodeLocalIdx, LON) = lonLat.lon();
787 nodesLonLat(nodeLocalIdx, LAT) = lonLat.lat();
788
789 // Set tij.
790 nodesTij(nodeLocalIdx, Coordinates::T) = localNode.t;
791 nodesTij(nodeLocalIdx, Coordinates::I) = localNode.i;
792 nodesTij(nodeLocalIdx, Coordinates::J) = localNode.j;
793
794 // Set halo.
795 nodesHalo(nodeLocalIdx) = localNode.halo;
796
797 // Set flags.
798 Topology::reset(nodesFlags(nodeLocalIdx));
799 switch (localNode.type) {
800 case ElemType::UNDEFINED: {
801 ATLAS_ASSERT(0, "Trying to add undefined node to mesh.");
802 break;
803 }
804 case ElemType::OWNER: {
805 nodesGhost(nodeLocalIdx) = 0;
806 // Vitally important that these two match!
807 ATLAS_ASSERT(nodeLocalIdx == localNode.globalPtr->remoteIdx,
808 "Owner local index and remote index do not match.");
809 break;
810 }
811 case ElemType::HALO: {
812 nodesGhost(nodeLocalIdx) = 1;
813 Topology::set(nodesFlags(nodeLocalIdx), Topology::GHOST);
814 break;
815 }
816 }
817
818 ++nodeLocalIdx;
819 }
820
821 // ---------------------------------------------------------------------------
822 // 7. ASSIGN CELLS TO MESH
823 // Again, we'll add t, i and j to mesh.cells(). We'll also add the xy and
824 // lonlat coordinates of the cell centres.
825 // ---------------------------------------------------------------------------
826
827 auto& cells = mesh.cells();
828
829 // Resize cells.
830 cells.add(mesh::ElementType::create("Quadrilateral"), static_cast<idx_t>(localCells.size()));
831
832 // Add extra fields.
833 tijField = cells.add(Field("tij", make_datatype<idx_t>(), make_shape(cells.size(), 3)));
834 tijField.set_variables(3);
835
836 Field xyField = cells.add(Field("xy", make_datatype<double>(), make_shape(cells.size(), 2)));
837 xyField.set_variables(2);
838
839 Field lonLatField = cells.add(Field("lonlat", make_datatype<double>(), make_shape(cells.size(), 2)));
840 lonLatField.set_variables(2);
841
842 Field ghostField = cells.add(Field("ghost", make_datatype<int>(), make_shape(cells.size())));
843
844 // Set field views.
845 auto cellsGlobalIdx = array::make_view<gidx_t, 1>(cells.global_index());
846 auto cellsRemoteIdx = array::make_indexview<idx_t, 1>(cells.remote_index());
847 auto cellsPart = array::make_view<int, 1>(cells.partition());
848 auto cellsGhost = array::make_view<int, 1>(ghostField);
849 auto cellsHalo = array::make_view<int, 1>(cells.halo());
850 auto cellsFlags = array::make_view<int, 1>(cells.flags());
851 auto cellsTij = array::make_view<idx_t, 2>(tijField);
852 auto cellsXy = array::make_view<double, 2>(xyField);
853 auto cellsLonLat = array::make_view<double, 2>(lonLatField);
854
855 // Set local cells.
856 auto& nodeConnectivity = cells.node_connectivity();
857 const idx_t cellElemIdx0 = cells.elements(0).begin();
858
859 // Set method to get node local index.
860 const auto getNodeLocalIdx = [&](idx_t i, idx_t j, idx_t t) -> idx_t {
861 const GlobalElem& globalNode = globalNodes[getNodeIdx(i, j, t)];
862
863 // Do some pointer arithmetic to get local index.
864 return static_cast<idx_t>(globalNode.localPtr - localNodes.data());
865 };
866
867 idx_t cellLocalIdx = 0;
868 for (const LocalElem& localCell : localCells) {
869 // Get local indices four surroundings nodes.
870 const auto quadNodeIdx = std::array<idx_t, 4>{getNodeLocalIdx(localCell.i, localCell.j, localCell.t),
871 getNodeLocalIdx(localCell.i + 1, localCell.j, localCell.t),
872 getNodeLocalIdx(localCell.i + 1, localCell.j + 1, localCell.t),
873 getNodeLocalIdx(localCell.i, localCell.j + 1, localCell.t)};
874
875 // Set connectivity.
876 nodeConnectivity.set(cellLocalIdx + cellElemIdx0, quadNodeIdx.data());
877
878 // Set global index.
879 cellsGlobalIdx(cellLocalIdx) = localCell.globalPtr->globalIdx;
880
881 // Set cell remote index.
882 cellsRemoteIdx(cellLocalIdx) = localCell.globalPtr->remoteIdx;
883
884 // Set partition.
885 cellsPart(cellLocalIdx) = localCell.globalPtr->part;
886
887 // Set xy.
888 const PointXY xyLocal = jacobian.xy(PointIJ(localCell.i + 0.5, localCell.j + 0.5), localCell.t);
889 const PointXY xyGlobal =
890 interiorCell(localCell.i, localCell.j) ? xyLocal : jacobian.xyLocalToGlobal(xyLocal, localCell.t).xy();
891
892 cellsXy(cellLocalIdx, XX) = xyLocal.x();
893 cellsXy(cellLocalIdx, YY) = xyLocal.y();
894
895 // Set lon-lat.
896 const PointLonLat lonLat = csProjection.lonlat(xyGlobal);
897 cellsLonLat(cellLocalIdx, LON) = lonLat.lon();
898 cellsLonLat(cellLocalIdx, LAT) = lonLat.lat();
899
900 // Set tij.
901 cellsTij(cellLocalIdx, Coordinates::T) = localCell.t;
902 cellsTij(cellLocalIdx, Coordinates::I) = localCell.i;
903 cellsTij(cellLocalIdx, Coordinates::J) = localCell.j;
904
905
906 // Set halo.
907 cellsHalo(cellLocalIdx) = localCell.halo;
908
909 // Set ghost.
910 cellsGhost(cellLocalIdx) = cellsHalo(cellLocalIdx) > 0;
911
912 // Set flags.
913 Topology::reset(cellsFlags(cellLocalIdx));
914 switch (localCell.type) {
915 case ElemType::UNDEFINED: {
916 ATLAS_ASSERT(0, "Trying to add undefined cell to mesh.");
917 break;
918 }
919 case ElemType::OWNER: {
920 // Vitally important that these two match!
921 ATLAS_ASSERT(cellLocalIdx == localCell.globalPtr->remoteIdx,
922 "Owner local index and remote index do not match.");
923 break;
924 }
925 case ElemType::HALO: {
926 Topology::set(cellsFlags(cellLocalIdx), Topology::GHOST);
927 break;
928 }
929 }
930
931 ++cellLocalIdx;
932 }
933
934 // ---------------------------------------------------------------------------
935 // 8. FINALISE
936 // Done. That was rather a lot of bookkeeping!
937 // ---------------------------------------------------------------------------
938
939 set_metadata(mesh);
940
941 return;
942 }
943
944 // -----------------------------------------------------------------------------
945
946 void CubedSphereMeshGenerator::set_metadata(Mesh& mesh) const {
947 const auto nHalo = options.get<int>("halo");
948
949 // Set basic halo metadata.
950 mesh.metadata().set("halo", nHalo);
951 mesh.metadata().set("halo_locked", true);
952 mesh.nodes().metadata().set("parallel", true);
953 mesh.cells().metadata().set("parallel", true);
954
955 // Loop over nodes and count number of halo elements.
956 auto nNodes = std::vector<idx_t>(nHalo + 1, 0);
957 const auto nodeHalo = array::make_view<int, 1>(mesh.nodes().halo());
958 for (idx_t i = 0; i < mesh.nodes().size(); ++i) {
959 ++nNodes[static_cast<size_t>(nodeHalo(i))];
960 }
961 std::partial_sum(nNodes.begin(), nNodes.end(), nNodes.begin());
962
963 // Set node halo metadata.
964 for (size_t i = 0; i < nNodes.size(); ++i) {
965 const auto str = "nb_nodes_including_halo[" + std::to_string(i) + "]";
966 mesh.metadata().set(str, nNodes[i]);
967 }
968 // Loop over cells and count number of halo elements.
969 auto nCells = std::vector<idx_t>(nHalo + 1, 0);
970 const auto cellHalo = array::make_view<int, 1>(mesh.cells().halo());
971 for (idx_t i = 0; i < mesh.cells().size(); ++i) {
972 ++nCells[static_cast<size_t>(cellHalo(i))];
973 }
974 std::partial_sum(nCells.begin(), nCells.end(), nCells.begin());
975
976 // Set cell halo metadata.
977 for (size_t i = 0; i < nCells.size(); ++i) {
978 const auto str = "nb_cells_including_halo[0][" + std::to_string(i) + "]";
979 mesh.metadata().set(str, nCells[i]);
980 }
981 }
982
983 // -----------------------------------------------------------------------------
984
985 void CubedSphereMeshGenerator::hash(eckit::Hash& h) const {
986 h.add("CubedSphereMeshGenerator");
987 options.hash(h);
988 }
989
990 // -----------------------------------------------------------------------------
991
992 namespace {
993 static MeshGeneratorBuilder<CubedSphereMeshGenerator> CubedSphereMeshGenerator(CubedSphereMeshGenerator::static_type());
994 }
995
996 // -----------------------------------------------------------------------------
997
998 } // namespace meshgenerator
999 } // namespace atlas