Debian Codegraph @

src/atlas/meshgenerator/detail/CubedSphereDualMeshGenerator.cc

raw
1 /*
2 * (C) Crown Copyright 2021 Met Office
3 *
4 * This software is licensed under the terms of the Apache Licence Version 2.0
5 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
6 */
7
8 #include <algorithm>
9 #include <cmath>
10 #include <limits>
11 #include <numeric>
12 #include <string>
13 #include <vector>
14
15 #include "eckit/utils/Hash.h"
16
17 #include "atlas/functionspace/CubedSphereColumns.h"
18 #include "atlas/grid/CubedSphereGrid.h"
19 #include "atlas/grid/Distribution.h"
20 #include "atlas/grid/Partitioner.h"
21 #include "atlas/library/config.h"
22 #include "atlas/mesh/Mesh.h"
23 #include "atlas/mesh/Nodes.h"
24 #include "atlas/meshgenerator/MeshGenerator.h"
25 #include "atlas/meshgenerator/detail/CubedSphereDualMeshGenerator.h"
26 #include "atlas/meshgenerator/detail/MeshGeneratorFactory.h"
27 #include "atlas/meshgenerator/detail/cubedsphere/CubedSphereUtility.h"
28 #include "atlas/parallel/mpi/mpi.h"
29
30 #define DEBUG_OUTPUT 0
31 #define DEBUG_OUTPUT_DETAIL 0
32
33 namespace atlas {
34 namespace meshgenerator {
35
36 // -----------------------------------------------------------------------------
37
38 CubedSphereDualMeshGenerator::CubedSphereDualMeshGenerator(const eckit::Parametrisation& p) {
39 // Get mpi_comm
40 std::string mpi_comm = mpi::comm().name();
41 p.get("mpi_comm", mpi_comm);
42 options.set("mpi_comm", mpi_comm);
43
44 configure_defaults();
45
46 // Get number of partitions.
47 size_t nb_parts;
48 if (p.get("nb_parts", nb_parts)) {
49 options.set("nb_parts", nb_parts);
50 }
51
52 // Get this partition.
53 int part;
54 if (p.get("part", part)) {
55 options.set("part", part);
56 }
57
58 // Get halo size.
59 int halo;
60 if (p.get("halo", halo)) {
61 options.set("halo", halo);
62 }
63
64 // Get partitioner.
65 std::string partitioner;
66 if (p.get("partitioner", partitioner) && partitioner.size()) {
67 options.set("partitioner", partitioner);
68 }
69 }
70
71 // -----------------------------------------------------------------------------
72
73
74 void CubedSphereDualMeshGenerator::configure_defaults() {
75 auto& comm = mpi::comm(options.getString("mpi_comm"));
76
77 // This option sets number of partitions.
78 options.set("nb_parts", comm.size());
79
80 // This option sets the part that will be generated.
81 options.set("part", comm.rank());
82
83 // This options sets the number of halo elements around each partition.
84 options.set("halo", 0);
85
86 // This options sets the default partitioner.
87 options.set<std::string>("partitioner", "cubedsphere");
88 }
89
90 // -----------------------------------------------------------------------------
91
92 void CubedSphereDualMeshGenerator::generate(const Grid& grid, Mesh& mesh) const {
93 // Get partitioner type and number of partitions from config.
94 const idx_t nParts = static_cast<idx_t>(options.get<size_t>("nb_parts"));
95 const std::string partType = options.get<std::string>("partitioner");
96
97 auto partConfig = util::Config{};
98 partConfig.set("type", partType);
99 partConfig.set("partitions", nParts);
100 partConfig.set("mpi_comm",options.getString("mpi_comm"));
101
102 // Use lonlat instead of xy for non cubedsphere partitioner.
103 if (partType != "cubedsphere") {
104 partConfig.set("coordinates", "lonlat");
105 }
106
107 // Set distribution.
108 mpi::Scope mpi_scope(options.getString("mpi_comm"));
109 const auto partitioner = grid::Partitioner(partConfig);
110 const auto distribution = grid::Distribution(grid, partitioner);
111
112 generate(grid, distribution, mesh);
113 }
114
115 // -----------------------------------------------------------------------------
116
117 void CubedSphereDualMeshGenerator::generate(const Grid& grid, const grid::Distribution& distribution,
118 Mesh& mesh) const {
119 // Check for correct grid and need for mesh
120 ATLAS_ASSERT(!mesh.generated());
121
122 // Cast grid to cubed sphere grid.
123 const auto csGrid = CubedSphereGrid(grid);
124
125 // Check for successful cast.
126 if (!csGrid) {
127 throw_Exception("CubedSphereDualMeshGenerator can only work with a cubedsphere grid.", Here());
128 }
129
130 // Check for correct grid stagger.
131 if (csGrid.stagger() != "C") {
132 throw_Exception("CubedSphereDualMeshGenerator will only work with a cell-centroid grid.", Here());
133 }
134
135 // Clone some grid properties.
136 setGrid(mesh, csGrid, distribution);
137
138 mpi::Scope mpi_scope(options.getString("mpi_comm"));
139 generate_mesh(csGrid, distribution, mesh);
140 }
141
142 // -----------------------------------------------------------------------------
143
144 namespace {
145
146 // (i, j) pair
147 class IJ {
148 public:
149 IJ(idx_t i, idx_t j): i_(i), j_(j) {}
150 idx_t i() const { return i_; }
151 idx_t j() const { return j_; }
152 IJ operator+(const IJ& ij) const { return IJ{i() + ij.i(), j() + ij.j()}; }
153 IJ operator-(const IJ& ij) const { return IJ{i() - ij.i(), j() - ij.j()}; }
154
155 private:
156 idx_t i_{};
157 idx_t j_{};
158 };
159
160 // Helper function to copy fields.
161 template <typename Value, int Rank>
162 void copyField(const Field& sourceField, Field& targetField) {
163 // Assign source field values to target field.
164 array::make_view<Value, Rank>(targetField).assign(array::make_view<Value, Rank>(sourceField));
165 }
166
167 // Get the surrounding node (i, j) pairs from a cell (i, j) pair.
168 std::vector<IJ> getIjNodes(const IJ& ijCell, idx_t N) {
169 // Rotate ij 90 degrees anitclockwise about ijPivot.
170 auto rotateAnticlockwise = [&](const IJ& ij, const IJ& ijPivot) {
171 const auto ijTemp = ij - ijPivot;
172 return IJ{-ijTemp.j(), ijTemp.i()} + ijPivot;
173 };
174
175 // Rotate ij 90 degrees clockwise about ijPivot.
176 auto rotateClockwise = [&](const IJ& ij, const IJ& ijPivot) {
177 const auto ijTemp = ij - ijPivot;
178 return IJ{ijTemp.j(), -ijTemp.i()} + ijPivot;
179 };
180
181 // Set standard surrounding nodes.
182 auto ijNodes = std::vector<IJ>{{ijCell.i() - 1, ijCell.j() - 1},
183 {ijCell.i(), ijCell.j() - 1},
184 {ijCell.i(), ijCell.j()},
185 {ijCell.i() - 1, ijCell.j()}};
186
187 // Modify nodes that lie in invalid corners of ij space.
188 // Either remove a node to make cell triangular, or rotate two of the
189 // nodes out of the forbidden space.
190
191 // Bottom-left corner.
192 if (ijCell.i() <= 0 && ijCell.j() <= 0) {
193 // Triangle.
194 if (ijCell.i() == 0 && ijCell.j() == 0) {
195 ijNodes.erase(ijNodes.begin());
196 }
197 // Quad (i)
198 else if (ijCell.i() == 0) {
199 ijNodes[0] = rotateClockwise(ijNodes[1], IJ{0, 0});
200 ijNodes[3] = rotateClockwise(ijNodes[2], IJ{0, 0});
201 }
202 else {
203 // Quad (ii)
204 ijNodes[0] = rotateAnticlockwise(ijNodes[3], IJ{0, 0});
205 ijNodes[1] = rotateAnticlockwise(ijNodes[2], IJ{0, 0});
206 }
207 }
208 // Bottom-right corner.
209 else if (ijCell.i() >= N && ijCell.j() <= 0) {
210 // Triangle.
211 if (ijCell.i() == N && ijCell.j() == 0) {
212 ijNodes.erase(ijNodes.begin() + 1);
213 }
214 // Quad (i)
215 else if (ijCell.j() == 0) {
216 ijNodes[0] = rotateClockwise(ijNodes[3], IJ{N - 1, 0});
217 ijNodes[1] = rotateClockwise(ijNodes[2], IJ{N - 1, 0});
218 }
219 // Quad (ii)
220 else {
221 ijNodes[1] = rotateAnticlockwise(ijNodes[0], IJ{N - 1, 0});
222 ijNodes[2] = rotateAnticlockwise(ijNodes[3], IJ{N - 1, 0});
223 }
224 }
225 // Top-right corner.
226 else if (ijCell.i() >= N && ijCell.j() >= N) {
227 // Triangle.
228 if (ijCell.i() == N && ijCell.j() == N) {
229 ijNodes.erase(ijNodes.begin() + 2);
230 }
231 // Quad (i)
232 else if (ijCell.i() == N) {
233 ijNodes[1] = rotateClockwise(ijNodes[0], IJ{N - 1, N - 1});
234 ijNodes[2] = rotateClockwise(ijNodes[3], IJ{N - 1, N - 1});
235 }
236 // Quad (ii)
237 else {
238 ijNodes[2] = rotateAnticlockwise(ijNodes[1], IJ{N - 1, N - 1});
239 ijNodes[3] = rotateAnticlockwise(ijNodes[0], IJ{N - 1, N - 1});
240 }
241 }
242 // Top-left corner.
243 else if (ijCell.i() <= 0 && ijCell.j() >= N) {
244 // Triangle.
245 if (ijCell.i() == 0 && ijCell.j() == N) {
246 std::vector<IJ> triangle {ijNodes[0], ijNodes[1], ijNodes[2]};
247 ijNodes.swap(triangle);
248 }
249 // Quad (i)
250 else if (ijCell.j() == N) {
251 ijNodes[2] = rotateClockwise(ijNodes[1], IJ{0, N - 1});
252 ijNodes[3] = rotateClockwise(ijNodes[0], IJ{0, N - 1});
253 }
254 // Quad (ii)
255 else {
256 ijNodes[0] = rotateAnticlockwise(ijNodes[1], IJ{0, N - 1});
257 ijNodes[3] = rotateAnticlockwise(ijNodes[2], IJ{0, N - 1});
258 }
259 }
260
261 return ijNodes;
262 }
263
264 } // namespace
265
266 void CubedSphereDualMeshGenerator::generate_mesh(const CubedSphereGrid& csGrid, const grid::Distribution& distribution,
267 Mesh& mesh) const {
268 ATLAS_TRACE("CubedSphereDualMeshGenerator::generate");
269
270 using Topology = atlas::mesh::Nodes::Topology;
271 using namespace detail::cubedsphere;
272
273 const idx_t N = csGrid.N();
274 const idx_t nHalo = options.get<int>("halo");
275
276 //--------------------------------------------------------------------------
277 // Create a cubed-sphere mesh.
278 //--------------------------------------------------------------------------
279
280 // Generate cubed sphere primal mesh.
281 auto primalOptions = options;
282 primalOptions.set("halo", nHalo + 1);
283 const auto primalMesh = MeshGenerator("cubedsphere", primalOptions).generate(csGrid, distribution);
284
285 // Generate fucntionspaces for cubed sphere primal mesh.
286 const auto primalCellsFunctionSpace = functionspace::CubedSphereCellColumns(primalMesh);
287 const auto primalNodesFunctionSpace = functionspace::CubedSphereNodeColumns(primalMesh);
288 const auto& primalCells = primalCellsFunctionSpace.cells();
289 const auto& primalNodes = primalNodesFunctionSpace.nodes();
290
291 //--------------------------------------------------------------------------
292 // Set dual mesh nodes (easy part).
293 //--------------------------------------------------------------------------
294
295 auto& nodes = mesh.nodes();
296 nodes.resize(primalCellsFunctionSpace.size());
297
298 nodes.add(Field("tij", array::make_datatype<idx_t>(), array::make_shape(nodes.size(), 3)));
299
300 // Copy mesh fields to dual mesh.
301 copyField<gidx_t, 1>(primalCells.global_index(), nodes.global_index());
302 copyField<idx_t, 1>(primalCells.remote_index(), nodes.remote_index());
303 copyField<int, 1>(primalCells.partition(), nodes.partition());
304 copyField<int, 1>(primalCells.halo(), nodes.halo());
305 copyField<int, 1>(primalCells.flags(), nodes.flags());
306 copyField<int, 1>(primalCells.field("ghost"), nodes.ghost());
307 copyField<double, 2>(primalCells.field("xy"), nodes.xy());
308 copyField<double, 2>(primalCells.field("lonlat"), nodes.lonlat());
309 copyField<idx_t, 2>(primalCells.field("tij"), nodes.field("tij"));
310
311 // Need to decrement halo by one.
312 auto nodesHalo = array::make_view<int, 1>(nodes.halo());
313
314 for (idx_t idx = 0; idx < nodes.size(); ++idx) {
315 nodesHalo(idx) = std::max(0, nodesHalo(idx) - 1);
316 }
317
318 //--------------------------------------------------------------------------
319 // Set dual mesh cells (not so easy part).
320 //--------------------------------------------------------------------------
321
322 // Make views to cubed sphere nodes.
323 const auto primalNodesHalo = array::make_view<int, 1>(primalNodes.halo());
324 const auto primalNodesTij = array::make_view<idx_t, 2>(primalNodes.field("tij"));
325
326 // Loop over all nodes of primal mesh, excluding outermost halo.
327 // Find dual mesh nodes around primal mesh nodes.
328 // Note: some halo cells near corners may be incomplete.
329
330 // Set of nodes around a cell.
331 struct NodeList {
332 std::vector<idx_t> nodes{}; // Node indices.
333 bool incomplete{}; // True if nodes are missing.
334 };
335
336 auto nodeLists = std::vector<NodeList>{};
337
338 for (idx_t idx = 0; idx < primalNodes.size(); ++idx) {
339 // Exclude outer ring of cubed sphere mesh halo.
340 if (primalNodesHalo(idx) == nHalo + 1) {
341 break;
342 }
343
344 nodeLists.emplace_back();
345 auto& nodeList = nodeLists.back();
346
347 // Get tij of cell.
348 const auto tCell = primalNodesTij(idx, Coordinates::T);
349 const auto ijCell = IJ{primalNodesTij(idx, Coordinates::I), primalNodesTij(idx, Coordinates::J)};
350
351 // Get ij of surrounding nodes.
352 auto ijNodes = getIjNodes(ijCell, N);
353
354 // Add indices to nodes vector.
355 for (const auto& ijNode : ijNodes) {
356 if (primalCellsFunctionSpace.is_valid_index(tCell, ijNode.i(), ijNode.j())) {
357 nodeList.nodes.push_back(primalCellsFunctionSpace.index(tCell, ijNode.i(), ijNode.j()));
358 }
359 else {
360 nodeList.incomplete = true;
361 }
362 }
363 }
364
365 // Figure out element types.
366 // Set first element type. ( first = type, second = count )
367 enum struct ElemType : size_t
368 {
369 LINE = 2,
370 TRIANGLE = 3,
371 QUADRILATERAL = 4
372 };
373 auto typeCounts =
374 std::vector<std::pair<ElemType, idx_t>>{std::make_pair(static_cast<ElemType>(nodeLists[0].nodes.size()), 1)};
375
376 // Count the number of consecutive lines, triangles or quadtrilaterals in dual mesh.
377 // This is an attempt to keep dual mesh cells in the same order as mesh nodes.
378 // Otherwise, the halo exchange bookkeeping is invalidated.
379 for (size_t idx = 1; idx < nodeLists.size(); ++idx) {
380 // Get the element type.
381 const auto elemType = static_cast<ElemType>(nodeLists[idx].nodes.size());
382
383 // Increment counter if this elemType is the same as last one
384 if (elemType == typeCounts.back().first) {
385 ++typeCounts.back().second;
386 }
387 // Otherwise add a new counter.
388 else {
389 typeCounts.emplace_back(elemType, 1);
390 }
391 }
392
393 // Add cells to mesh.
394 auto& cells = mesh.cells();
395 idx_t nCells{};
396 // Loop through type counters.
397 for (const auto& typeCount : typeCounts) {
398 // Select element type.
399 switch (typeCount.first) {
400 // Add a block of lines.
401 case ElemType::LINE: {
402 cells.add(mesh::ElementType::create("Line"), typeCount.second);
403 break;
404 }
405 // Add a block of triangles.
406 case ElemType::TRIANGLE: {
407 cells.add(mesh::ElementType::create("Triangle"), typeCount.second);
408 break;
409 }
410 // Add a block of quadrilaterals.
411 case ElemType::QUADRILATERAL: {
412 cells.add(mesh::ElementType::create("Quadrilateral"), typeCount.second);
413 break;
414 }
415 default: {
416 ATLAS_THROW_EXCEPTION("Unknown element type with " +
417 std::to_string(static_cast<size_t>(typeCount.first)) + " nodes.");
418 break;
419 }
420 }
421 // Increment the total number of cells.
422 nCells += typeCount.second;
423 }
424
425 // Add extra fields to cells.
426 cells.add(Field("tij", array::make_datatype<idx_t>(), array::make_shape(cells.size(), 3)));
427 cells.add(Field("xy", array::make_datatype<double>(), array::make_shape(cells.size(), 2)));
428 cells.add(Field("lonlat", array::make_datatype<double>(), array::make_shape(cells.size(), 2)));
429 cells.add(Field("ghost", array::make_datatype<int>(), array::make_shape(cells.size())));
430
431 // Copy dual cells fields from nodes.
432 copyField<gidx_t, 1>(primalNodes.global_index(), cells.global_index());
433 copyField<idx_t, 1>(primalNodes.remote_index(), cells.remote_index());
434 copyField<int, 1>(primalNodes.partition(), cells.partition());
435 copyField<int, 1>(primalNodes.ghost(), cells.field("ghost"));
436 copyField<int, 1>(primalNodes.halo(), cells.halo());
437 copyField<int, 1>(primalNodes.flags(), cells.flags());
438 copyField<idx_t, 2>(primalNodes.field("tij"), cells.field("tij"));
439 copyField<double, 2>(primalNodes.xy(), cells.field("xy"));
440 copyField<double, 2>(primalNodes.lonlat(), cells.field("lonlat"));
441
442 // Get view of flags field.
443 auto dualCellsFlags = array::make_view<int, 1>(cells.flags());
444
445 // Get node connectivity.
446 auto& nodeConnectivity = cells.node_connectivity();
447
448 // Loop over dual mesh cells and set connectivity.
449 for (idx_t idx = 0; idx < nCells; ++idx) {
450 // Set connectivity.
451 nodeConnectivity.set(idx, nodeLists[idx].nodes.data());
452
453 // Set invalid flag if cell is incomplete.
454 if (nodeLists[idx].incomplete) {
455 Topology::set(dualCellsFlags(idx), Topology::INVALID);
456 }
457 }
458
459 // Set metadata.
460 set_metadata(mesh);
461
462 return;
463 }
464
465 // -----------------------------------------------------------------------------
466
467 void CubedSphereDualMeshGenerator::set_metadata(Mesh& mesh) const {
468 const auto nHalo = options.get<int>("halo");
469
470 // Set basic halo metadata.
471 mesh.metadata().set("halo", nHalo);
472 mesh.metadata().set("halo_locked", true);
473 mesh.nodes().metadata().set("parallel", true);
474 mesh.cells().metadata().set("parallel", true);
475
476 mesh.metadata().set("mpi_comm",options.getString("mpi_comm"));
477
478
479 // Loop over nodes and count number of halo elements.
480 auto nNodes = std::vector<idx_t>(nHalo + 2, 0);
481 const auto nodeHalo = array::make_view<int, 1>(mesh.nodes().halo());
482 for (idx_t i = 0; i < mesh.nodes().size(); ++i) {
483 ++nNodes[static_cast<size_t>(nodeHalo(i))];
484 }
485 std::partial_sum(nNodes.begin(), nNodes.end(), nNodes.begin());
486
487 // Set node halo metadata.
488 for (size_t i = 0; i < nNodes.size(); ++i) {
489 const auto str = "nb_nodes_including_halo[" + std::to_string(i) + "]";
490 mesh.metadata().set(str, nNodes[i]);
491 }
492
493
494 // Loop over cells and count number of halo elements.
495 auto nCells = std::vector<std::vector<idx_t>>(mesh.cells().nb_types(), std::vector<idx_t>(nHalo + 1, 0));
496 const auto cellHalo = array::make_view<int, 1>(mesh.cells().halo());
497
498 for (idx_t i = 0; i < mesh.cells().nb_types(); ++i) {
499 const auto& elems = mesh.cells().elements(i);
500 for (idx_t j = elems.begin(); j < elems.end(); ++j) {
501 ++nCells[static_cast<size_t>(i)][static_cast<size_t>(cellHalo(j))];
502 }
503 std::partial_sum(nCells[static_cast<size_t>(i)].begin(), nCells[static_cast<size_t>(i)].end(),
504 nCells[static_cast<size_t>(i)].begin());
505 }
506
507 // Set cell halo metadata.
508 for (size_t i = 0; i < nCells.size(); ++i) {
509 for (size_t j = 0; j < nCells[i].size(); ++j) {
510 const auto str = "nb_cells_including_halo[" + std::to_string(i) + "][" + std::to_string(j) + "]";
511 mesh.metadata().set(str, nCells[i][j]);
512 }
513 }
514 }
515
516 // -----------------------------------------------------------------------------
517
518 void CubedSphereDualMeshGenerator::hash(eckit::Hash& h) const {
519 h.add("CubedSphereDualMeshGenerator");
520 options.hash(h);
521 }
522
523 // -----------------------------------------------------------------------------
524
525 namespace {
526 static MeshGeneratorBuilder<CubedSphereDualMeshGenerator> CubedSphereDualMeshGenerator(
527 CubedSphereDualMeshGenerator::static_type());
528 }
529
530 // -----------------------------------------------------------------------------
531
532 } // namespace meshgenerator
533 } // namespace atlas