Debian Codegraph @

src/atlas/meshgenerator/detail/RegularMeshGenerator.cc

raw
1 /*
2 * (C) Copyright 2013 ECMWF.
3 *
4 * This software is licensed under the terms of the Apache Licence Version 2.0
5 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
6 * In applying this licence, ECMWF does not waive the privileges and immunities
7 * granted to it by virtue of its status as an intergovernmental organisation
8 * nor does it submit to any jurisdiction.
9 */
10
11 #include <algorithm>
12 #include <cmath>
13 #include <limits>
14 #include <numeric>
15 #include <vector>
16
17 #include "eckit/utils/Hash.h"
18
19 #include "atlas/array/Array.h"
20 #include "atlas/array/ArrayView.h"
21 #include "atlas/array/IndexView.h"
22 #include "atlas/field/Field.h"
23 #include "atlas/grid/Distribution.h"
24 #include "atlas/grid/Partitioner.h"
25 #include "atlas/grid/StructuredGrid.h"
26 #include "atlas/library/config.h"
27 #include "atlas/mesh/ElementType.h"
28 #include "atlas/mesh/Elements.h"
29 #include "atlas/mesh/HybridElements.h"
30 #include "atlas/mesh/Mesh.h"
31 #include "atlas/mesh/Nodes.h"
32 #include "atlas/meshgenerator/detail/MeshGeneratorFactory.h"
33 #include "atlas/meshgenerator/detail/RegularMeshGenerator.h"
34 #include "atlas/parallel/mpi/mpi.h"
35 #include "atlas/runtime/Exception.h"
36 #include "atlas/runtime/Log.h"
37 #include "atlas/util/CoordinateEnums.h"
38
39 #define DEBUG_OUTPUT 0
40 #define DEBUG_OUTPUT_DETAIL 0
41
42 using Topology = atlas::mesh::Nodes::Topology;
43
44 namespace atlas {
45 namespace meshgenerator {
46
47 RegularMeshGenerator::RegularMeshGenerator(const eckit::Parametrisation& p) {
48
49 std::string mpi_comm = mpi::comm().name();
50 p.get("mpi_comm", mpi_comm);
51 options.set("mpi_comm",mpi_comm);
52
53 configure_defaults();
54
55 // options copied from Structured MeshGenerator
56 size_t nb_parts;
57 if (p.get("nb_parts", nb_parts)) {
58 options.set("nb_parts", nb_parts);
59 }
60
61 size_t part;
62 if (p.get("part", part)) {
63 options.set("part", part);
64 }
65
66 std::string partitioner;
67 if (p.get("partitioner", partitioner)) {
68 if (not grid::Partitioner::exists(partitioner)) {
69 Log::warning() << "Atlas does not have support for partitioner " << partitioner << ". "
70 << "Defaulting to use partitioner EqualRegions" << std::endl;
71 partitioner = "equal_regions";
72 }
73 options.set("partitioner", partitioner);
74 }
75
76 // options specifically for this MeshGenerator
77 bool periodic_x;
78 if (p.get("periodic_x", periodic_x)) {
79 options.set("periodic_x", periodic_x);
80 }
81
82 bool periodic_y;
83 if (p.get("periodic_y", periodic_y)) {
84 options.set("periodic_y", periodic_y);
85 }
86
87 bool biperiodic;
88 if (p.get("biperiodic", biperiodic)) {
89 options.set("periodic_x", biperiodic);
90 options.set("periodic_y", biperiodic);
91 }
92 }
93
94 void RegularMeshGenerator::configure_defaults() {
95 std::string mpi_comm;
96 options.get("mpi_comm",mpi_comm);
97 auto& comm = mpi::comm(mpi_comm);
98
99 // This option sets number of parts the mesh will be split in
100 options.set("nb_parts", comm.size());
101
102 // This option sets the part that will be generated
103 options.set("part", comm.rank());
104
105 // This options sets the default partitioner
106 std::string partitioner;
107 if (grid::Partitioner::exists("ectrans") && comm.size() > 1) {
108 partitioner = "ectrans";
109 }
110 else {
111 partitioner = "checkerboard";
112 }
113 options.set<std::string>("partitioner", partitioner);
114
115 // Options for for periodic grids
116 options.set<bool>("periodic_x", false);
117 options.set<bool>("periodic_y", false);
118 }
119
120 void RegularMeshGenerator::generate(const Grid& grid, Mesh& mesh) const {
121 ATLAS_ASSERT(!mesh.generated());
122
123 const RegularGrid rg = RegularGrid(grid);
124 if (!rg) {
125 throw_Exception("RegularMeshGenerator can only work with a Regular grid", Here());
126 }
127
128 size_t nb_parts = options.get<size_t>("nb_parts");
129
130 std::string partitioner_type = "checkerboard";
131 options.get("checkerboard", partitioner_type);
132
133 // if ( rg->nlat()%2 == 1 ) partitioner_factory = "equal_regions"; // Odd
134 // number of latitudes
135 // if ( nb_parts == 1 || eckit::mpi::size() == 1 ) partitioner_factory =
136 // "equal_regions"; // Only one part --> Trans is slower
137
138 mpi::scope::push(options.getString("mpi_comm"));
139 grid::Partitioner partitioner(partitioner_type, nb_parts);
140 grid::Distribution distribution(partitioner.partition(grid));
141 mpi::scope::pop();
142 generate(grid, distribution, mesh);
143 }
144
145 void RegularMeshGenerator::hash(eckit::Hash& h) const {
146 h.add("RegularMeshGenerator");
147 options.hash(h);
148 }
149
150 void RegularMeshGenerator::generate(const Grid& grid, const grid::Distribution& distribution, Mesh& mesh) const {
151 const auto rg = RegularGrid(grid);
152 if (!rg) {
153 throw_Exception("Grid could not be cast to a Regular", Here());
154 }
155
156 ATLAS_ASSERT(!mesh.generated());
157
158 if (grid.size() != static_cast<idx_t>(distribution.size())) {
159 std::stringstream msg;
160 msg << "Number of points in grid (" << grid.size()
161 << ") different from "
162 "number of points in grid distribution ("
163 << distribution.size() << ")";
164 throw_AssertionFailed(msg.str(), Here());
165 }
166
167 // clone some grid properties
168 setGrid(mesh, rg, distribution);
169
170 generate_mesh(rg, distribution, mesh);
171 }
172
173 void RegularMeshGenerator::generate_mesh(const RegularGrid& rg, const grid::Distribution& distribution,
174 // const Region& region,
175 Mesh& mesh) const {
176 mpi::Scope mpi_scope(options.getString("mpi_comm"));
177 int mypart = options.get<size_t>("part");
178 int nparts = options.get<size_t>("nb_parts");
179 int nx = rg.nx();
180 int ny = rg.ny();
181 const int y_numbering = (rg.y().front() < rg.y().back()) ? +1 : -1;
182
183 bool periodic_x = options.get<bool>("periodic_x") or rg.periodic();
184 bool periodic_y = options.get<bool>("periodic_y");
185
186 Log::debug() << Here() << " periodic_x = " << periodic_x << std::endl;
187 Log::debug() << Here() << " periodic_y = " << periodic_y << std::endl;
188
189 // for asynchronous output
190 #if DEBUG_OUTPUT
191 sleep(mypart);
192 #endif
193
194 // this function should do the following:
195 // - define nodes with
196 // mesh.nodes().resize(nnodes);
197 // mesh::Nodes& nodes = mesh.nodes();
198 // following properties should be defined:
199 // array::ArrayView<double,2> xy ( nodes.xy() );
200 // array::ArrayView<gidx_t,1> glb_idx ( nodes.global_index() );
201 // array::ArrayView<int, 1> part ( nodes.partition() );
202 // array::ArrayView<int, 1> ghost ( nodes.ghost() );
203 // array::ArrayView<int, 1> flags ( nodes.flags() );
204 // - define cells (only quadrilaterals for now) with
205 // mesh.cells().add(mesh::ElementType::create("Quadrilateral"), nquads );
206 // further define cells with
207 // array::ArrayView<gidx_t,1> cells_glb_idx( mesh.cells().global_index()
208 // );
209 // array::ArrayView<int,1> cells_part( mesh.cells().partition() );
210 // - define connectivity with
211 // mesh::HybridElements::Connectivity& node_connectivity =
212 // mesh.cells().node_connectivity();
213 // node_connectivity.set( jcell, quad_nodes );
214 // where quad_nodes is a 4-element integer array containing the LOCAL
215 // indices of the nodes
216
217 // Start with calculating number of quadrilaterals
218 // The rule do determine if a cell belongs to a proc is the following: if the
219 // lowerleft corner of the cell belongs to that proc.
220 // so we loop over all gridpoints, select those that belong to the proc, and
221 // determine the number of cells
222 int ii_glb; // global index
223 int ncells;
224
225 // vector of local indices: necessary for remote indices of ghost nodes
226 std::vector<int> local_idx(rg.size(), -1);
227 std::vector<int> current_idx(nparts, 0); // index counter for each proc
228
229 // determine rectangle (ix_min:ix_max) x (iy_min:iy_max) surrounding the nodes
230 // on this processor
231 int ix_min, ix_max, iy_min, iy_max, ix_glb, iy_glb, ix, iy;
232 int nnodes_nonghost, nnodes; // number of nodes: non-ghost; total; inside
233 // surrounding rectangle
234 int nnodes_SR, ii;
235
236 // loop over all points to determine local indices and surroundig rectangle
237 ix_min = nx + 1;
238 ix_max = 0;
239 iy_min = ny + 1;
240 iy_max = 0;
241 nnodes_nonghost = 0;
242
243 ii_glb = 0;
244 for (iy = 0; iy < ny; iy++) {
245 for (ix = 0; ix < nx; ix++) {
246 local_idx[ii_glb] = current_idx[distribution.partition(ii_glb)]++; // store local index on
247 // the local proc of
248 // this point
249 if (distribution.partition(ii_glb) == mypart) {
250 ++nnodes_nonghost; // non-ghost node: belongs to this part
251 ix_min = std::min(ix_min, ix);
252 ix_max = std::max(ix_max, ix);
253 iy_min = std::min(iy_min, iy);
254 iy_max = std::max(iy_max, iy);
255 }
256 ++ii_glb; // global index
257 }
258 }
259
260 // add one row/column for ghost nodes (which include periodicity points)
261 ix_max = ix_max + 1;
262 iy_max = iy_max + 1;
263
264 #if DEBUG_OUTPUT_DETAIL
265 std::cout << "[" << mypart << "] : "
266 << "SR = " << ix_min << ":" << ix_max << " x " << iy_min << ":" << iy_max << std::endl;
267 #endif
268
269 // dimensions of surrounding rectangle (SR)
270 int nxl = ix_max - ix_min + 1;
271 int nyl = iy_max - iy_min + 1;
272
273 // upper estimate for number of nodes
274 nnodes_SR = nxl * nyl;
275
276 // partitions and local indices in SR
277 std::vector<int> parts_SR(nnodes_SR, -1);
278 std::vector<int> local_idx_SR(nnodes_SR, -1);
279 std::vector<bool> is_ghost_SR(nnodes_SR, true);
280 ii = 0; // index inside SR
281 for (iy = 0; iy < nyl; iy++) {
282 iy_glb = (iy_min + iy); // global y-index
283 for (ix = 0; ix < nxl; ix++) {
284 ix_glb = (ix_min + ix); // global x-index
285 is_ghost_SR[ii] = !((parts_SR[ii] == mypart) && ix < nxl - 1 && iy < nyl - 1);
286 if (ix_glb < nx && iy_glb < ny) {
287 ii_glb = (iy_glb)*nx + ix_glb; // global index
288 parts_SR[ii] = distribution.partition(ii_glb);
289 local_idx_SR[ii] = local_idx[ii_glb];
290 is_ghost_SR[ii] = !((parts_SR[ii] == mypart) && ix < nxl - 1 && iy < nyl - 1);
291 }
292 else if (ix_glb == nx && iy_glb < ny) {
293 // take properties from the point to the left
294 parts_SR[ii] = distribution.partition(iy_glb * nx + ix_glb - 1);
295 local_idx_SR[ii] = -1;
296 is_ghost_SR[ii] = true;
297 }
298 else if (iy_glb == ny && ix_glb < nx) {
299 // take properties from the point below
300 parts_SR[ii] = distribution.partition((iy_glb - 1) * nx + ix_glb);
301 local_idx_SR[ii] = -1;
302 is_ghost_SR[ii] = true;
303 }
304 else {
305 // take properties from the point belowleft
306 parts_SR[ii] = distribution.partition((iy_glb - 1) * nx + ix_glb - 1);
307 local_idx_SR[ii] = -1;
308 is_ghost_SR[ii] = true;
309 }
310 ++ii;
311 }
312 }
313
314 #if DEBUG_OUTPUT_DETAIL
315 std::cout << "[" << mypart << "] : "
316 << "parts_SR = ";
317 for (ii = 0; ii < nnodes_SR; ii++)
318 std::cout << parts_SR[ii] << ",";
319 std::cout << std::endl;
320 std::cout << "[" << mypart << "] : "
321 << "local_idx_SR = ";
322 for (ii = 0; ii < nnodes_SR; ii++)
323 std::cout << local_idx_SR[ii] << ",";
324 std::cout << std::endl;
325 std::cout << "[" << mypart << "] : "
326 << "is_ghost_SR = ";
327 for (ii = 0; ii < nnodes_SR; ii++)
328 std::cout << is_ghost_SR[ii] << ",";
329 std::cout << std::endl;
330 #endif
331
332 // vectors marking nodes that are necessary for this proc's cells
333 std::vector<bool> is_node_SR(nnodes_SR, false);
334
335 // determine number of cells and number of nodes
336 nnodes = 0;
337 ncells = 0;
338 for (iy = 0; iy < nyl - 1; iy++) { // don't loop into ghost/periodicity row
339 for (ix = 0; ix < nxl - 1; ix++) { // don't loop into ghost/periodicity column
340 ii = iy * nxl + ix;
341 if (!is_ghost_SR[ii]) {
342 // mark this node as being used
343 if (!is_node_SR[ii]) {
344 ++nnodes;
345 is_node_SR[ii] = true;
346 }
347 // check if this node is the lowerleft corner of a new cell
348 if ((ix_min + ix < nx - 1 || periodic_x) && (iy_min + iy < ny - 1 || periodic_y)) {
349 ++ncells;
350 // mark lowerright corner
351 ii = iy * nxl + ix + 1;
352 if (!is_node_SR[ii]) {
353 ++nnodes;
354 is_node_SR[ii] = true;
355 }
356 // mark upperleft corner
357 ii = (iy + 1) * nxl + ix;
358 if (!is_node_SR[ii]) {
359 ++nnodes;
360 is_node_SR[ii] = true;
361 }
362 // mark upperright corner
363 ii = (iy + 1) * nxl + ix + 1;
364 if (!is_node_SR[ii]) {
365 ++nnodes;
366 is_node_SR[ii] = true;
367 }
368 }
369 // periodic points are always needed, even if they don't belong to a
370 // cell
371 ii = iy * nxl + ix + 1;
372 if (periodic_x && ix_min + ix == nx - 1 && !is_node_SR[ii]) {
373 ++nnodes;
374 is_node_SR[ii] = true;
375 }
376 ii = (iy + 1) * nxl + ix;
377 if (periodic_y && iy_min + iy == ny - 1 && !is_node_SR[ii]) {
378 ++nnodes;
379 is_node_SR[ii] = true;
380 }
381 ii = (iy + 1) * nxl + ix + 1;
382 if (periodic_x && periodic_y && ix_min + ix == nx - 1 && iy_min + iy == ny - 1 && !is_node_SR[ii]) {
383 ++nnodes;
384 is_node_SR[ii] = true;
385 }
386 }
387 }
388 }
389
390 #if DEBUG_OUTPUT_DETAIL
391 std::cout << "[" << mypart << "] : "
392 << "nnodes = " << nnodes << std::endl;
393 std::cout << "[" << mypart << "] : "
394 << "is_node_SR = ";
395 for (ii = 0; ii < nnodes_SR; ii++)
396 std::cout << is_node_SR[ii] << ",";
397 std::cout << std::endl;
398 #endif
399
400 // define nodes and associated properties
401 mesh.nodes().resize(nnodes);
402 mesh::Nodes& nodes = mesh.nodes();
403 auto xy = array::make_view<double, 2>(nodes.xy());
404 auto lonlat = array::make_view<double, 2>(nodes.lonlat());
405 auto glb_idx = array::make_view<gidx_t, 1>(nodes.global_index());
406 auto remote_idx = array::make_indexview<idx_t, 1>(nodes.remote_index());
407 auto part = array::make_view<int, 1>(nodes.partition());
408 auto ghost = array::make_view<int, 1>(nodes.ghost());
409 auto flags = array::make_view<int, 1>(nodes.flags());
410
411 // define cells and associated properties
412 mesh.cells().add(mesh::ElementType::create("Quadrilateral"), ncells);
413 int quad_begin = mesh.cells().elements(0).begin();
414 auto cells_part = array::make_view<int, 1>(mesh.cells().partition());
415 mesh::HybridElements::Connectivity& node_connectivity = mesh.cells().node_connectivity();
416
417 idx_t quad_nodes[4];
418 int jcell = quad_begin;
419 int inode, inode_nonghost, inode_ghost;
420
421 // global indices for periodicity points
422 inode = nx * ny;
423 std::vector<int> glb_idx_px(ny + 1, -1);
424 std::vector<int> glb_idx_py(nx + 1, -1);
425 if (periodic_x) {
426 for (iy = 0; iy < ny + (periodic_y ? 1 : 0); iy++) {
427 glb_idx_px[iy] = inode++;
428 }
429 }
430 if (periodic_y) {
431 for (ix = 0; ix < nx; ix++) {
432 glb_idx_py[ix] = inode++;
433 }
434 }
435
436 // loop over nodes and set properties
437 ii = 0;
438 inode_nonghost = 0;
439 inode_ghost = nnodes_nonghost; // ghost nodes start counting after nonghost nodes
440 for (iy = 0; iy < nyl; iy++) {
441 for (ix = 0; ix < nxl; ix++) {
442 // node properties
443 if (is_node_SR[ii]) {
444 // set node counter
445 if (is_ghost_SR[ii]) {
446 inode = inode_ghost++;
447 }
448 else {
449 inode = inode_nonghost++;
450 }
451 // global index
452 ix_glb = (ix_min + ix); // don't take modulus here: periodicity points
453 // have their own global index.
454 iy_glb = (iy_min + iy);
455 if (ix_glb < nx && iy_glb < ny) {
456 ii_glb = iy_glb * nx + ix_glb; // no periodic point
457 }
458 else {
459 if (ix_glb == nx) {
460 // periodicity point in x-direction
461 ii_glb = glb_idx_px[iy_glb];
462 }
463 else {
464 // periodicity point in x-direction
465 ii_glb = glb_idx_py[ix_glb];
466 }
467 }
468 glb_idx(inode) = ii_glb + 1; // starting from 1
469 // grid coordinates
470 double _xy[2];
471 if (iy_glb < ny) {
472 // normal calculation
473 rg.xy(ix_glb, iy_glb, _xy);
474 }
475 else {
476 // for periodic_y grids, iy_glb==ny lies outside the range of
477 // latitudes in the Structured grid...
478 // so we extrapolate from two other points -- this is okay for regular
479 // grids with uniform spacing.
480 double xy1[2], xy2[2];
481 rg.xy(ix_glb, iy_glb - 1, xy1);
482 rg.xy(ix_glb, iy_glb - 2, xy2);
483 _xy[0] = 2 * xy1[0] - xy2[0];
484 _xy[1] = 2 * xy1[1] - xy2[1];
485 }
486 xy(inode, LON) = _xy[LON];
487 xy(inode, LAT) = _xy[LAT];
488
489 // geographic coordinates by using projection
490 rg.projection().xy2lonlat(_xy);
491 lonlat(inode, LON) = _xy[LON];
492 lonlat(inode, LAT) = _xy[LAT];
493
494 // part
495 part(inode) = parts_SR[ii];
496 // ghost nodes
497 ghost(inode) = is_ghost_SR[ii];
498 // flags
499 Topology::reset(flags(inode));
500 if (ghost(inode)) {
501 Topology::set(flags(inode), Topology::GHOST);
502 remote_idx(inode) = local_idx_SR[ii];
503 // change local index -- required for cells
504 local_idx_SR[ii] = inode;
505 }
506 else {
507 remote_idx(inode) = -1;
508 }
509
510 #if DEBUG_OUTPUT_DETAIL
511 std::cout << "[" << mypart << "] : "
512 << "New node "
513 << "\n\t";
514 std::cout << "[" << mypart << "] : "
515 << "\tinode=" << inode << "; ix_glb=" << ix_glb << "; iy_glb=" << iy_glb
516 << "; glb_idx=" << ii_glb << std::endl;
517 std::cout << "[" << mypart << "] : "
518 << "\tglon=" << lonlat(inode, 0) << "; glat=" << lonlat(inode, 1)
519 << "; glb_idx=" << glb_idx(inode) << std::endl;
520 #endif
521 }
522 ++ii;
523 }
524 }
525
526 // loop over nodes and define cells
527 for (iy = 0; iy < nyl - 1; iy++) { // don't loop into ghost/periodicity row
528 for (ix = 0; ix < nxl - 1; ix++) { // don't loop into ghost/periodicity column
529 ii = iy * nxl + ix;
530 if (!is_ghost_SR[ii]) {
531 if ((ix_min + ix < nx - 1 || periodic_x) && (iy_min + iy < ny - 1 || periodic_y)) {
532 // define cell corners (local indices)
533 if (y_numbering < 0) {
534 quad_nodes[0] = local_idx_SR[ii];
535 quad_nodes[3] = local_idx_SR[iy * nxl + ix + 1]; // point to the right
536 quad_nodes[2] = local_idx_SR[(iy + 1) * nxl + ix + 1]; // point above right
537 quad_nodes[1] = local_idx_SR[(iy + 1) * nxl + ix]; // point above
538 }
539 else {
540 quad_nodes[0] = local_idx_SR[ii];
541 quad_nodes[1] = local_idx_SR[iy * nxl + ix + 1]; // point to the right
542 quad_nodes[2] = local_idx_SR[(iy + 1) * nxl + ix + 1]; // point above right
543 quad_nodes[3] = local_idx_SR[(iy + 1) * nxl + ix]; // point above
544
545 }
546 node_connectivity.set(jcell, quad_nodes);
547 cells_part(jcell) = mypart;
548 #if DEBUG_OUTPUT_DETAIL
549 std::cout << "[" << mypart << "] : "
550 << "New quad " << jcell << "\n\t";
551 std::cout << "[" << mypart << "] : " << quad_nodes[0] << "," << quad_nodes[1] << ","
552 << quad_nodes[2] << "," << quad_nodes[3] << std::endl;
553 #endif
554 ++jcell;
555 }
556 }
557 }
558 }
559
560 #if DEBUG_OUTPUT
561 // list nodes
562 for (inode = 0; inode < nnodes; inode++) {
563 std::cout << "[" << mypart << "] : "
564 << " node " << inode << ": ghost = " << ghost(inode) << ", glb_idx = " << glb_idx(inode) - 1
565 << ", part = " << part(inode) << ", lon = " << lonlat(inode, 0) << ", lat = " << lonlat(inode, 1)
566 << ", remote_idx = " << remote_idx(inode) << std::endl;
567 }
568
569 int* cell_nodes;
570 for (jcell = 0; jcell < ncells; jcell++) {
571 std::cout << "[" << mypart << "] : "
572 << " cell " << jcell << ": " << node_connectivity(jcell, 0) << "," << node_connectivity(jcell, 1)
573 << "," << node_connectivity(jcell, 2) << "," << node_connectivity(jcell, 3) << std::endl;
574 }
575 #endif
576
577 mesh.metadata().set("nb_parts",options.getInt("nb_parts"));
578 mesh.metadata().set("part",options.getInt("part"));
579 mesh.metadata().set("mpi_comm",options.getString("mpi_comm"));
580
581 generateGlobalElementNumbering(mesh);
582
583 nodes.metadata().set("parallel", true);
584 }
585
586 namespace {
587 static MeshGeneratorBuilder<RegularMeshGenerator> __RegularMeshGenerator("regular");
588 }
589
590 } // namespace meshgenerator
591 } // namespace atlas