Debian Codegraph @

src/atlas/meshgenerator/detail/NodalCubedSphereMeshGenerator.cc

raw
1 /*
2 * (C) Copyright 2020 UCAR
3 *
4 * This software is licensed under the terms of the Apache Licence Version 2.0
5 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
6 */
7
8 #include <algorithm>
9 #include <cmath>
10 #include <limits>
11 #include <numeric>
12 #include <vector>
13
14 #include "eckit/utils/Hash.h"
15
16 #include "atlas/field/Field.h"
17 #include "atlas/grid/CubedSphereGrid.h"
18 #include "atlas/grid/Distribution.h"
19 #include "atlas/grid/Partitioner.h"
20 #include "atlas/library/config.h"
21 #include "atlas/mesh/Connectivity.h"
22 #include "atlas/mesh/ElementType.h"
23 #include "atlas/mesh/Elements.h"
24 #include "atlas/mesh/HybridElements.h"
25 #include "atlas/mesh/Mesh.h"
26 #include "atlas/mesh/Nodes.h"
27 #include "atlas/meshgenerator/detail/MeshGeneratorFactory.h"
28 #include "atlas/meshgenerator/detail/NodalCubedSphereMeshGenerator.h"
29 #include "atlas/meshgenerator/detail/cubedsphere/CubedSphereUtility.h"
30 #include "atlas/parallel/mpi/mpi.h"
31 #include "atlas/runtime/Exception.h"
32 #include "atlas/runtime/Log.h"
33 #include "atlas/util/Constants.h"
34 #include "atlas/util/CoordinateEnums.h"
35
36 #define DEBUG_OUTPUT 0
37 #define DEBUG_OUTPUT_DETAIL 0
38
39 using Topology = atlas::mesh::Nodes::Topology;
40
41 namespace atlas {
42 namespace meshgenerator {
43
44 // -------------------------------------------------------------------------------------------------
45
46 NodalCubedSphereMeshGenerator::NodalCubedSphereMeshGenerator(const eckit::Parametrisation& p) {}
47
48 // -------------------------------------------------------------------------------------------------
49
50 void NodalCubedSphereMeshGenerator::configure_defaults() {}
51
52 // -------------------------------------------------------------------------------------------------
53
54 void NodalCubedSphereMeshGenerator::generate(const Grid& grid, Mesh& mesh) const {
55 // Check for proper grid and need for mesh
56 ATLAS_ASSERT(!mesh.generated());
57 const CubedSphereGrid csg = CubedSphereGrid(grid);
58 if (!csg) {
59 throw_Exception("NodalCubedSphereMeshGenerator can only work with a cubedsphere grid", Here());
60 }
61
62 // Number of processors
63 //size_t nb_parts = options.get<size_t>( "nb_parts" );
64
65 // Decomposition type
66 std::string partitioner_type = "checkerboard";
67 options.get("checkerboard", partitioner_type);
68
69 // Partitioner
70 grid::Partitioner partitioner(partitioner_type, 1);
71 grid::Distribution distribution(partitioner.partition(grid));
72
73 generate(grid, distribution, mesh);
74 }
75
76 // -------------------------------------------------------------------------------------------------
77
78 void NodalCubedSphereMeshGenerator::generate(const Grid& grid, const grid::Distribution& distribution,
79 Mesh& mesh) const {
80 const auto csgrid = CubedSphereGrid(grid);
81
82 using namespace detail::cubedsphere;
83
84 const int N = csgrid.N();
85 const int nTiles = csgrid.tiles().size();
86
87 // N must be greater than 1.
88 if (N < 2) {
89 throw_Exception("N must be greater than 1 for NodalCubedSphereMeshGenerator", Here());
90 }
91
92 // grid must have node staggering.
93 if (csgrid.stagger() != "L") {
94 throw_Exception(
95 "NodalCubedSphereMeshGenerator will only work with a"
96 "nodal grid. Try CubedSphereMeshGenerator instead.",
97 Here());
98 }
99
100 // Get tiles
101 const auto& csprojection = csgrid.cubedSphereProjection();
102 // grid must use FV3Tiles class.
103 if (csprojection.getCubedSphereTiles().type() != "cubedsphere_fv3") {
104 throw_Exception("NodalCubedSphereMeshGenerator only works with FV3 tiles", Here());
105 }
106
107 // Make a list linking ghost (t, i, j) values to known (t, i, j)
108 // This will be different for each cube-sphere once MeshGenerator is generalised.
109
110 using Tij = typename std::array<idx_t, 3>;
111
112 // first: ghost (t, i, j); second owned (t, i, j).
113 using TijPair = typename std::pair<Tij, Tij>;
114
115 auto ghostToOwnedTij = std::vector<TijPair>{};
116
117
118 // -------------------------------------------------------------------------
119 // BEGIN FV3 SPECIFIC MAP
120 // -------------------------------------------------------------------------
121
122 // Special points.
123 ghostToOwnedTij.push_back(TijPair{{0, N, N}, {2, 0, 0}});
124 ghostToOwnedTij.push_back(TijPair{{1, N, N}, {3, 0, 0}});
125 ghostToOwnedTij.push_back(TijPair{{2, N, N}, {4, 0, 0}});
126 ghostToOwnedTij.push_back(TijPair{{3, N, N}, {5, 0, 0}});
127 ghostToOwnedTij.push_back(TijPair{{4, N, N}, {0, 0, 0}});
128 ghostToOwnedTij.push_back(TijPair{{5, N, N}, {1, 0, 0}});
129 ghostToOwnedTij.push_back(TijPair{{2, 0, N}, {0, 0, N}});
130 ghostToOwnedTij.push_back(TijPair{{4, 0, N}, {0, 0, N}});
131 ghostToOwnedTij.push_back(TijPair{{3, N, 0}, {1, N, 0}});
132 ghostToOwnedTij.push_back(TijPair{{5, N, 0}, {1, N, 0}});
133
134 // Tile 1
135 for (idx_t ix = 1; ix < N; ix++) {
136 ghostToOwnedTij.push_back(TijPair{{0, ix, N}, {2, 0, N - ix}});
137 }
138 for (idx_t iy = 0; iy < N; iy++) {
139 ghostToOwnedTij.push_back(TijPair{{0, N, iy}, {1, 0, iy}});
140 }
141
142 // Tile 2
143 for (idx_t ix = 0; ix < N; ix++) {
144 ghostToOwnedTij.push_back(TijPair{{1, ix, N}, {2, ix, 0}});
145 }
146 for (idx_t iy = 1; iy < N; iy++) {
147 ghostToOwnedTij.push_back(TijPair{{1, N, iy}, {3, N - iy, 0}});
148 }
149
150 // Tile 3
151 for (idx_t ix = 1; ix < N; ix++) {
152 ghostToOwnedTij.push_back(TijPair{{2, ix, N}, {4, 0, N - ix}});
153 }
154 for (idx_t iy = 0; iy < N; iy++) {
155 ghostToOwnedTij.push_back(TijPair{{2, N, iy}, {3, 0, iy}});
156 }
157
158 // Tile 4
159 for (idx_t ix = 0; ix < N; ix++) {
160 ghostToOwnedTij.push_back(TijPair{{3, ix, N}, {4, ix, 0}});
161 }
162 for (idx_t iy = 1; iy < N; iy++) {
163 ghostToOwnedTij.push_back(TijPair{{3, N, iy}, {5, N - iy, 0}});
164 }
165
166 // Tile 5
167 for (idx_t ix = 1; ix < N; ix++) {
168 ghostToOwnedTij.push_back(TijPair{{4, ix, N}, {0, 0, N - ix}});
169 }
170 for (idx_t iy = 0; iy < N; iy++) {
171 ghostToOwnedTij.push_back(TijPair{{4, N, iy}, {5, 0, iy}});
172 }
173
174 // Tile 6
175 for (idx_t ix = 0; ix < N; ix++) {
176 ghostToOwnedTij.push_back(TijPair{{5, ix, N}, {0, ix, 0}});
177 }
178 for (idx_t iy = 1; iy < N; iy++) {
179 ghostToOwnedTij.push_back(TijPair{{5, N, iy}, {1, N - iy, 0}});
180 }
181
182
183 // -------------------------------------------------------------------------
184 // END FV3 SPECIFIC MAP
185 // -------------------------------------------------------------------------
186
187 ATLAS_TRACE("NodalCubedSphereMeshGenerator::generate");
188 Log::debug() << "Number of faces per tile edge = " << std::to_string(N) << std::endl;
189
190 // Number of nodes
191 int nnodes = nTiles * N * N + 2; // Number of unique grid nodes
192 int nnodes_all = nTiles * (N + 1) * (N + 1); // Number of grid nodes including edge and corner duplicates
193 int ncells = nTiles * N * N; // Number of unique grid cells
194
195 // Construct mesh nodes
196 // --------------------
197 mesh.nodes().resize(nnodes_all);
198 mesh::Nodes& nodes = mesh.nodes();
199 auto xy = array::make_view<double, 2>(nodes.xy());
200 auto lonlat = array::make_view<double, 2>(nodes.lonlat());
201 auto glb_idx = array::make_view<gidx_t, 1>(nodes.global_index());
202 auto remote_idx = array::make_indexview<idx_t, 1>(nodes.remote_index());
203 auto part = array::make_view<int, 1>(nodes.partition());
204 auto ghost = array::make_view<int, 1>(nodes.ghost());
205 auto flags = array::make_view<int, 1>(nodes.flags());
206
207 // Loop over entire grid
208 // ---------------------
209
210 // Node array will include duplicates of the grid points to make it easier to fill up the
211 // neighbours. However the mesh will not contain these points so no Ghost nodes are required.
212 // We could include the duplicate points in the array if we make them Ghost nodes but it is not
213 // clear if this provides any benefit.
214
215 array::ArrayT<int> NodeArrayT(nTiles, N + 1, N + 1); // All grid points including duplicates
216 auto NodeArray = array::make_view<int, 3>(NodeArrayT);
217
218 // Add owned nodes to node array
219 idx_t nOwned = 0;
220 auto addOwnedNode = [&](Tij tijOwned) {
221 // Set node array
222 NodeArray(tijOwned[0], tijOwned[1], tijOwned[2]) = nOwned;
223
224 // Get xy from global xy grid array
225 double xy_[3];
226 csgrid.xy(tijOwned[1], tijOwned[2], tijOwned[0], xy_);
227
228 xy(nOwned, XX) = xy_[XX];
229 xy(nOwned, YY) = xy_[YY];
230
231 // Get lonlat from global lonlat array
232 double lonlat_[2];
233 csgrid.lonlat(tijOwned[1], tijOwned[2], tijOwned[0], lonlat_);
234
235 lonlat(nOwned, LON) = lonlat_[LON];
236 lonlat(nOwned, LAT) = lonlat_[LAT];
237
238 // Is not ghost node
239 mesh::Nodes::Topology::reset(flags(nOwned));
240 ghost(nOwned) = 0;
241
242 glb_idx(nOwned) = nOwned + 1;
243 remote_idx(nOwned) = nOwned;
244 part(nOwned) = distribution.partition(nOwned);
245
246 ++nOwned;
247
248 return;
249 };
250
251 // Loop over owned (t, i, j)
252
253 for (auto& p : csgrid.tij()) {
254 addOwnedNode(Tij{p.t(), p.i(), p.j()});
255 }
256
257 // Assert that the correct number of nodes have been set
258 ATLAS_ASSERT(nnodes == nOwned, "Insufficient nodes");
259
260 // Vector of ghost global index of each ghost point
261 auto ghostGblIdx = std::vector<idx_t>();
262 auto ownedGblIdx = std::vector<idx_t>();
263
264 // Add ghost nodes to node array
265 // (nGhost started after nOwned)
266 auto nGhost = nOwned;
267 auto addGhostNode = [&](Tij tijGhost, Tij tijOwned) {
268 // Get concrete node id
269 auto nOwned = NodeArray(tijOwned[0], tijOwned[1], tijOwned[2]);
270
271 // Add ghost node to NodeArray
272 NodeArray(tijGhost[0], tijGhost[1], tijGhost[2]) = nGhost;
273
274 // "Create" ghost xy coordinate.
275
276 // Get Jacobian of coords rtw indices
277 auto x0 = xy(NodeArray(tijGhost[0], 0, 0), XX);
278 auto y0 = xy(NodeArray(tijGhost[0], 0, 0), YY);
279
280 auto dx_di = xy(NodeArray(tijGhost[0], 1, 0), XX) - x0;
281 auto dx_dj = xy(NodeArray(tijGhost[0], 0, 1), XX) - x0;
282 auto dy_di = xy(NodeArray(tijGhost[0], 1, 0), YY) - y0;
283 auto dy_dj = xy(NodeArray(tijGhost[0], 0, 1), YY) - y0;
284
285 // Set xy coordinates
286 xy(nGhost, XX) = x0 + tijGhost[1] * dx_di + tijGhost[2] * dx_dj;
287 xy(nGhost, YY) = y0 + tijGhost[1] * dy_di + tijGhost[2] * dy_dj;
288
289 // Same lonlat as concrete points
290 lonlat(nGhost, LON) = lonlat(nOwned, LON);
291 lonlat(nGhost, LAT) = lonlat(nOwned, LAT);
292
293 // Is ghost node
294 mesh::Nodes::Topology::set(flags(nGhost), mesh::Nodes::Topology::GHOST);
295 ghost(nGhost) = 1;
296
297 // Partitioning logic to be added in future PR
298 glb_idx(nGhost) = nGhost + 1;
299 remote_idx(nGhost) = nGhost;
300
301 // not sure (below - for multiple PEs)
302 part(nGhost) = distribution.partition(nOwned);
303
304 // Append metadata
305 // Global indicies of ghost node and owned node
306 ghostGblIdx.push_back(nGhost + 1);
307 ownedGblIdx.push_back(nOwned + 1);
308
309 ++nGhost;
310 };
311
312 // Loop over ghost (t, i, j)
313 for (auto& pPair : ghostToOwnedTij) {
314 addGhostNode(pPair.first, pPair.second);
315 }
316
317
318 // Assert that the correct number of nodes have been set when duplicates are added
319 ATLAS_ASSERT(nnodes_all == nGhost, "Insufficient nodes");
320
321 for (idx_t it = 0; it < nTiles; it++) {
322 for (idx_t ix = 0; ix < N + 1; ix++) {
323 for (idx_t iy = 0; iy < N + 1; iy++) {
324 ATLAS_ASSERT(NodeArray(it, ix, iy) != -9999, "Node Array Not Set Properly");
325 }
326 }
327 }
328
329 // Cells in mesh
330 mesh.cells().add(mesh::ElementType::create("Quadrilateral"), nTiles * N * N);
331 //int quad_begin = mesh.cells().elements( 0 ).begin();
332 auto cells_part = array::make_view<int, 1>(mesh.cells().partition());
333 auto cells_gidx = array::make_view<gidx_t, 1>(mesh.cells().global_index());
334 auto cells_ridx = array::make_indexview<idx_t, 1>(mesh.cells().remote_index());
335 atlas::mesh::HybridElements::Connectivity& node_connectivity = mesh.cells().node_connectivity();
336
337 int icell = 0;
338 idx_t quad_nodes[4];
339
340 for (int it = 0; it < nTiles; it++) {
341 for (int ix = 0; ix < N; ix++) {
342 for (int iy = 0; iy < N; iy++) {
343 quad_nodes[0] = NodeArray(it, ix, iy);
344 quad_nodes[1] = NodeArray(it, ix + 1, iy);
345 quad_nodes[2] = NodeArray(it, ix + 1, iy + 1);
346 quad_nodes[3] = NodeArray(it, ix, iy + 1);
347
348 node_connectivity.set(icell, quad_nodes);
349 cells_part(icell) = part(quad_nodes[0]);
350 cells_gidx(icell) = icell + 1;
351 cells_ridx(icell) = icell;
352
353 ++icell;
354 }
355 }
356 }
357
358 // Assertion that correct number of cells are set
359 ATLAS_ASSERT(ncells == icell, "Insufficient cells have been set");
360
361 // Parallel
362 generateGlobalElementNumbering(mesh);
363 nodes.metadata().set("parallel", true);
364
365 // Global indices of ghost nodes.
366 nodes.metadata().set("ghost-global-idx", ghostGblIdx);
367
368 // Global indices of owned nodes for each ghost node (same order as above)
369 nodes.metadata().set("owned-global-idx", ownedGblIdx);
370 }
371
372 // -------------------------------------------------------------------------------------------------
373
374 void NodalCubedSphereMeshGenerator::hash(eckit::Hash& h) const {
375 h.add("NodalCubedSphereMeshGenerator");
376 options.hash(h);
377 }
378
379 // -------------------------------------------------------------------------------------------------
380
381 namespace {
382 static MeshGeneratorBuilder<NodalCubedSphereMeshGenerator> NodalCubedSphereMeshGenerator(
383 NodalCubedSphereMeshGenerator::static_type());
384 }
385
386 // -------------------------------------------------------------------------------------------------
387
388 } // namespace meshgenerator
389 } // namespace atlas