Debian Codegraph @

packages/seacas/applications/zellij/Grid.C

raw
1 // Copyright(C) 2021, 2022, 2023, 2024 National Technology & Engineering Solutions
2 // of Sandia, LLC (NTESS). Under the terms of Contract DE-NA0003525 with
3 // NTESS, the U.S. Government retains certain rights in this software.
4 //
5 // See packages/seacas/LICENSE for details
6
7 #include <cstdlib>
8 #include <numeric>
9
10 #include "Cell.h"
11 #include "Decompose.h"
12 #include "Grid.h"
13 #include "ZE_SystemInterface.h"
14 #include "ZE_Version.h"
15 #include <Ioss_CommSet.h>
16 #include <Ioss_ElementBlock.h>
17 #include <Ioss_IOFactory.h>
18 #include <Ioss_NodeBlock.h>
19 #include <Ioss_ParallelUtils.h>
20 #include <Ioss_Region.h>
21 #include <Ioss_SideBlock.h>
22 #include <Ioss_SideSet.h>
23 #include <Ioss_SmartAssert.h>
24
25 #include <exodusII.h>
26 #include <fmt/chrono.h>
27 #include <fmt/color.h>
28 #include <fmt/ostream.h>
29 #include <fmt/ranges.h>
30 #include <open_file_limit.h>
31 #include <time_stamp.h>
32 #include <tokenize.h>
33
34 //! \file
35
36 extern unsigned int debug_level;
37
38 #if __cplusplus == 201103L
39 // From: https://stackoverflow.com/questions/17902405/how-to-implement-make-unique-function-in-c11
40 namespace std {
41 template <class T> struct _Unique_if
42 {
43 typedef unique_ptr<T> _Single_object;
44 };
45
46 template <class T> struct _Unique_if<T[]>
47 {
48 typedef unique_ptr<T[]> _Unknown_bound;
49 };
50
51 template <class T, size_t N> struct _Unique_if<T[N]>
52 {
53 typedef void _Known_bound;
54 };
55
56 template <class T, class... Args>
57 typename _Unique_if<T>::_Single_object make_unique(Args &&...args)
58 {
59 return unique_ptr<T>(new T(std::forward<Args>(args)...));
60 }
61
62 template <class T> typename _Unique_if<T>::_Unknown_bound make_unique(size_t n)
63 {
64 typedef typename remove_extent<T>::type U;
65 return unique_ptr<T>(new U[n]());
66 }
67
68 template <class T, class... Args>
69 typename _Unique_if<T>::_Known_bound make_unique(Args &&...) = delete;
70 } // namespace std
71 #endif
72
73 namespace {
74 std::string tsFormat = "[{:%H:%M:%S}]";
75 int axis_index(const std::string &axis_str)
76 {
77 char axis = axis_str[0];
78 if (axis == 'x' || axis == 'i') {
79 return 0;
80 }
81 if (axis == 'X' || axis == 'I') {
82 return 1;
83 }
84 if (axis == 'y' || axis == 'j') {
85 return 2;
86 }
87 if (axis == 'Y' || axis == 'J') {
88 return 3;
89 }
90 if (axis == 'z' || axis == 'k') {
91 return 4;
92 }
93 if (axis == 'Z' || axis == 'K') {
94 return 5;
95 }
96 return -1;
97 }
98
99 unsigned int which_sidesets(const std::string &sideset_string)
100 {
101 unsigned generate = 0;
102 for (const auto &c : sideset_string) {
103 if (c == 'x' || c == 'i') {
104 generate |= (unsigned)Flg::MIN_I;
105 }
106 if (c == 'y' || c == 'j') {
107 generate |= (unsigned)Flg::MIN_J;
108 }
109 if (c == 'z' || c == 'k') {
110 generate |= (unsigned)Flg::MIN_K;
111 }
112 if (c == 'X' || c == 'I') {
113 generate |= (unsigned)Flg::MAX_I;
114 }
115 if (c == 'Y' || c == 'J') {
116 generate |= (unsigned)Flg::MAX_J;
117 }
118 if (c == 'Z' || c == 'K') {
119 generate |= (unsigned)Flg::MAX_K;
120 }
121 }
122 return generate;
123 }
124
125 Ioss::PropertyManager parse_properties(SystemInterface &interFace, int int_size);
126
127 std::shared_ptr<Ioss::Region> create_input_region(const std::string &key, std::string filename,
128 bool ints_32_bits);
129
130 template <typename INT>
131 std::vector<INT> generate_node_map(Grid &grid, const Cell &cell, Mode mode, INT /*dummy*/);
132
133 void output_summary(Ioss::Region *region, std::ostream &strm);
134 size_t handle_elements(Grid &grid, int start_rank, int rank_count);
135 size_t handle_nodes(Grid &grid, int start_rank, int rank_count);
136 void handle_communications(Grid &grid, int start_rank, int rank_count);
137 void handle_surfaces(Grid &grid, int start_rank, int rank_count);
138 void generate_surfaces(Grid &grid, int start_rank, int rank_count);
139 } // namespace
140
141 Grid::Grid(SystemInterface &interFace)
142 : m_offset(interFace.offset()), m_scaleFactor(interFace.scale_factor()),
143 m_parallelSize(interFace.ranks()), m_rankCount(interFace.rank_count()),
144 m_startRank(interFace.start_rank()), m_equivalenceNodes(interFace.equivalence_nodes()),
145 m_useInternalSidesets(!interFace.ignore_internal_sidesets()),
146 m_subCycle(interFace.subcycle()), m_minimizeOpenFiles(interFace.minimize_open_files()),
147 m_generatedSideSets(which_sidesets(interFace.sideset_surfaces()))
148 {
149 set_sideset_names(interFace.sideset_names());
150 }
151
152 void Grid::set_extent(size_t extent_i, size_t extent_j, size_t /* unused */)
153 {
154 m_gridI = extent_i;
155 m_gridJ = extent_j;
156 m_grid.resize(m_gridI * m_gridJ);
157 }
158
159 bool Grid::initialize(size_t i, size_t j, const std::string &key)
160 {
161 if (unit_cells().find(key) == unit_cells().end()) {
162 return false;
163 }
164 const auto &unit_cell = unit_cells()[key];
165 SMART_ASSERT(unit_cell->m_region != nullptr)(i)(j)(key);
166
167 auto &cell = get_cell(i, j);
168 cell.initialize(i, j, unit_cell);
169 return true;
170 }
171
172 void Grid::add_unit_cell(const std::string &key, const std::string &unit_filename, bool ints32bit)
173 {
174 static size_t open_files = open_file_limit();
175 if (!minimize_open_files(Minimize::UNIT) && unit_cells().size() >= open_files) {
176 // Just hit the limit... Close all previous unit_cell files and set the minimize_open_files
177 // behavior to UNIT.
178 for (const auto &unit_cell : m_unitCells) {
179 unit_cell.second->m_region->get_database()->closeDatabase();
180 }
181 fmt::print(stderr, fmt::fg(fmt::color::yellow),
182 "\nWARNING: Number of unit cells exceeds open file limit. Setting "
183 "'mimimize_open_file' mode to UNIT.\n\n");
184 m_minimizeOpenFiles = Minimize((unsigned)m_minimizeOpenFiles | (unsigned)Minimize::UNIT);
185 }
186
187 if (unit_cells().find(key) != unit_cells().end()) {
188 fmt::print(stderr, fmt::fg(fmt::color::red),
189 "\nERROR: There is a duplicate `unit cell` ({}) in the lattice dictionary.\n\n",
190 key);
191 exit(EXIT_FAILURE);
192 }
193
194 auto region = create_input_region(key, unit_filename, ints32bit);
195
196 if (region == nullptr) {
197 fmt::print(stderr, fmt::fg(fmt::color::red),
198 "\nERROR: Unable to open the database '{}' associated with the unit cell '{}'.\n\n",
199 unit_filename, key);
200 exit(EXIT_FAILURE);
201 }
202
203 unit_cells().emplace(key, std::make_shared<UnitCell>(region));
204 if (minimize_open_files(Minimize::UNIT)) {
205 unit_cells()[key]->m_region->get_database()->closeDatabase();
206 }
207
208 if (debug_level & 2) {
209 m_pu.progress(fmt::format("\tCreated Unit Cell {}", key));
210 }
211 }
212
213 void Grid::set_coordinate_offsets()
214 {
215 // All unit_cells have same X, Y (and Z) extent. Only need X and Y
216 auto x_range = get_cell(0, 0).get_coordinate_range(Axis::X);
217 auto y_range = get_cell(0, 0).get_coordinate_range(Axis::Y);
218 auto delta_x = x_range.second - x_range.first;
219 auto delta_y = y_range.second - y_range.first;
220
221 for (size_t j = 0; j < JJ(); j++) {
222 for (size_t i = 0; i < II(); i++) {
223 auto &cell = get_cell(i, j);
224 cell.m_offX = delta_x * (double)i;
225 cell.m_offY = delta_y * (double)j;
226
227 if (debug_level & 2) {
228 util().progress(
229 fmt::format("\tCell({}, {}) X = {}, Y = {}", i, j, cell.m_offX, cell.m_offY));
230 }
231 }
232 }
233 }
234
235 void Grid::create_output_regions(SystemInterface &interFace)
236 {
237 if (debug_level & 2) {
238 util().progress(__func__);
239 }
240 m_outputRegions.resize(interFace.ranks());
241
242 int int_size = interFace.ints32bit() ? 4 : 8;
243 Ioss::PropertyManager properties = parse_properties(interFace, int_size);
244 if (parallel_size() == 1) {
245 properties.add(Ioss::Property("OMIT_EXODUS_NUM_MAPS", 1));
246 }
247 // Disable this for now. Readers need to be modified and propagated to allow this.
248 // properties.add(Ioss::Property("MINIMAL_NEMESIS_DATA", 1));
249
250 if (debug_level & 2) {
251 properties.add(Ioss::Property("ENABLE_TRACING", 1));
252 }
253
254 for (int i = m_startRank; i < m_startRank + m_rankCount; i++) {
255 // Define the output database(s)...
256 if (m_parallelSize > 1) {
257 properties.add(Ioss::Property("processor_count", parallel_size()));
258 properties.add(Ioss::Property("my_processor", i));
259 }
260 Ioss::DatabaseIO *dbo =
261 Ioss::IOFactory::create("exodus", interFace.outputName_, Ioss::WRITE_RESTART,
262 Ioss::ParallelUtils::comm_self(), properties);
263 if (dbo == nullptr || !dbo->ok(true)) {
264 std::exit(EXIT_FAILURE);
265 }
266 m_outputRegions[i] = std::make_unique<Ioss::Region>(dbo, "zellij_output_region");
267 output_region(i)->begin_mode(Ioss::STATE_DEFINE_MODEL);
268 output_region(i)->property_add(Ioss::Property("code_name", qainfo[0]));
269 output_region(i)->property_add(Ioss::Property("code_version", qainfo[2]));
270
271 if (interFace.minimize_open_files() == Minimize::OUTPUT ||
272 interFace.minimize_open_files() == Minimize::ALL) {
273 output_region(i)->get_database()->closeDatabase();
274 }
275 }
276 }
277
278 void Grid::set_sideset_names(const std::string &names)
279 {
280 // 'names' is a string of the form "Axis:name,Axis:name, ..."
281 // For example "x:left,y:top,X:right,Y:bottom"
282 // Parse the list and only update the names specified in the list...
283
284 if (names.empty()) {
285 return;
286 }
287
288 auto tokens = Ioss::tokenize(names, ",");
289 for (auto &token : tokens) {
290 auto axis = token.substr(0, 1);
291 bool valid = axis.find_first_not_of("ijkIJKxyzXYZ") == std::string::npos;
292 if (!valid) {
293 fmt::print(stderr, fmt::fg(fmt::color::red),
294 "\nERROR: Invalid axis '{}' specified for sideset name. Valid is one of "
295 "'ijkIJKxyzXYZ'.\n\n",
296 axis);
297 exit(EXIT_FAILURE);
298 }
299
300 // Now get the name...
301 auto ss_name = token.substr(2);
302
303 // Update the name in the list of generated sideset names...
304 auto index = axis_index(axis);
305 SMART_ASSERT(index >= 0)(axis)(index);
306 generated_surface_names[index] = std::move(ss_name);
307 }
308 }
309
310 void Grid::decompose(const std::string &method)
311 {
312 if (debug_level & 2) {
313 util().progress(__func__);
314 }
315 decompose_grid(*this, m_parallelSize, method);
316
317 categorize_processor_boundaries();
318 }
319
320 void Grid::categorize_processor_boundaries()
321 {
322 if (debug_level & 2) {
323 util().progress(__func__);
324 }
325
326 // Now iterate the cells and tell each cell the rank of all neighboring cells...
327 // boundary with its "left" or "lower" neighboring cell.
328
329 for (size_t j = 0; j < JJ(); j++) {
330 for (size_t i = 0; i < II(); i++) {
331 auto &cell = get_cell(i, j);
332 if (i > 0) {
333 const auto &left = get_cell(i - 1, j);
334 cell.set_rank(Loc::L, left.rank(Loc::C));
335 if (j > 0) {
336 const auto &BL = get_cell(i - 1, j - 1);
337 cell.set_rank(Loc::BL, BL.rank(Loc::C));
338 }
339 if (j < JJ() - 1) {
340 const auto &TL = get_cell(i - 1, j + 1);
341 cell.set_rank(Loc::TL, TL.rank(Loc::C));
342 }
343 }
344 if (i < II() - 1) {
345 const auto &right = get_cell(i + 1, j);
346 cell.set_rank(Loc::R, right.rank(Loc::C));
347 if (j > 0) {
348 const auto &BR = get_cell(i + 1, j - 1);
349 cell.set_rank(Loc::BR, BR.rank(Loc::C));
350 }
351 if (j < JJ() - 1) {
352 const auto &TR = get_cell(i + 1, j + 1);
353 cell.set_rank(Loc::TR, TR.rank(Loc::C));
354 }
355 }
356 if (j > 0) {
357 const auto &B = get_cell(i, j - 1);
358 cell.set_rank(Loc::B, B.rank(Loc::C));
359 }
360 if (j < JJ() - 1) {
361 const auto &T = get_cell(i, j + 1);
362 cell.set_rank(Loc::T, T.rank(Loc::C));
363 }
364 }
365 }
366
367 if (debug_level & 32) {
368 auto width = Ioss::Utils::number_width(parallel_size());
369 for (size_t j = 0; j < JJ(); j++) {
370 for (size_t i = 0; i < II(); i++) {
371 const auto &cell = get_cell(i, j);
372 auto left = cell.processor_boundary(Loc::L) ? '<' : ' ';
373 auto below = cell.processor_boundary(Loc::B) ? '^' : ' ';
374 fmt::print(" {0}{1:{3}}{2}", left, cell.rank(Loc::C), below, width);
375 }
376 fmt::print("\n");
377 }
378 }
379 }
380
381 void Grid::generate_sidesets()
382 {
383 if (m_generatedSideSets != 0) {
384 for (const auto &unit_cell : m_unitCells) {
385 unit_cell.second->generate_boundary_faces(m_generatedSideSets);
386 }
387 }
388 }
389
390 void Grid::internal_process()
391 {
392 if (debug_level & 2) {
393 util().progress(__func__);
394 }
395 auto node_count = handle_nodes(*this, m_startRank, m_rankCount);
396 auto element_count = handle_elements(*this, m_startRank, m_rankCount);
397 handle_communications(*this, m_startRank, m_rankCount);
398 if (m_useInternalSidesets) {
399 handle_surfaces(*this, m_startRank, m_rankCount);
400 }
401 generate_surfaces(*this, m_startRank, m_rankCount);
402
403 for (int i = m_startRank; i < m_startRank + m_rankCount; i++) {
404 output_region(i)->end_mode(Ioss::STATE_DEFINE_MODEL);
405 if (debug_level & 64) {
406 output_summary(output_region(i), std::cerr);
407 }
408 }
409 if (util().parallel_rank() == 0) {
410 fmt::print(" {} Nodes; {} Elements.\n", fmt::group_digits(node_count),
411 fmt::group_digits(element_count));
412 }
413 }
414
415 template void Grid::process(SystemInterface &, int64_t);
416 template void Grid::process(SystemInterface &, int);
417
418 template <typename INT> void Grid::process(SystemInterface &interFace, INT /* dummy */)
419 {
420 bool subcycle = m_subCycle;
421 int start_rank = m_startRank;
422 int rank_count = m_rankCount;
423 int end_rank = subcycle ? m_parallelSize : m_startRank + m_rankCount;
424
425 if (end_rank > m_parallelSize) {
426 end_rank = m_parallelSize;
427 m_rankCount = m_parallelSize - m_startRank;
428 }
429
430 for (int begin = start_rank; begin < end_rank; begin += rank_count) {
431 m_startRank = begin;
432 if (m_startRank + m_rankCount > m_parallelSize) {
433 m_rankCount = m_parallelSize - m_startRank;
434 }
435 if (debug_level & 2) {
436 fmt::print(stderr, "{} Processing Ranks {} to {}\n", time_stamp(tsFormat), begin,
437 begin + rank_count - 1);
438 }
439
440 create_output_regions(interFace);
441
442 internal_process();
443 if (debug_level & 2) {
444 fmt::print(stderr, "{} Lattice Processing Finalized\n", time_stamp(tsFormat));
445 }
446
447 output_model(INT(0));
448 if (debug_level & 2) {
449 fmt::print(stderr, "{} Model Output\n", time_stamp(tsFormat));
450 }
451 }
452 }
453
454 template void Grid::output_model(int64_t);
455 template void Grid::output_model(int);
456
457 // Write the output database(s) for all ranks...
458 template <typename INT> void Grid::output_model(INT /*dummy*/)
459 {
460 if (debug_level & 2) {
461 util().progress(__func__);
462 }
463 // Coordinates do not depend on order of operations, so can do these
464 // a rank at a time which can be more efficient once we exceed
465 // maximum number of open files... Also easier to parallelize...
466 for (int r = m_startRank; r < m_startRank + m_rankCount; r++) {
467 for (size_t j = 0; j < JJ(); j++) {
468 for (size_t i = 0; i < II(); i++) {
469 const auto &cell = get_cell(i, j);
470 if (cell.rank(Loc::C) == r) {
471 output_nodal_coordinates(cell);
472 }
473 }
474 }
475 if (minimize_open_files(Minimize::OUTPUT)) {
476 output_region(r)->get_database()->closeDatabase();
477 }
478 }
479 if (debug_level & 2) {
480 util().progress("\tEnd Nodal Coordinate Output");
481 }
482
483 // Surfaces do not depend on order of operations, so can do these
484 // a rank at a time which can be more efficient once we exceed
485 // maximum number of open files... Also easier to parallelize...
486 for (int r = m_startRank; r < m_startRank + m_rankCount; r++) {
487 for (size_t j = 0; j < JJ(); j++) {
488 for (size_t i = 0; i < II(); i++) {
489 auto &cell = get_cell(i, j);
490 if (cell.rank(Loc::C) == r) {
491 if (m_useInternalSidesets) {
492 output_surfaces(cell, INT(0));
493 }
494 output_generated_surfaces(cell, INT(0));
495 }
496 }
497 }
498 if (minimize_open_files(Minimize::OUTPUT)) {
499 output_region(r)->get_database()->closeDatabase();
500 }
501 }
502 if (debug_level & 2) {
503 util().progress("\tEnd Surface Output");
504 }
505
506 // All the rest of these depend on progressing through the cells in
507 // correct order so that can pass information correctly from cell to
508 // cell. Need to figure out how to eliminate this ordering so can
509 // parallelize and not worry about open file count.
510
511 for (size_t j = 0; j < JJ(); j++) {
512 for (size_t i = 0; i < II(); i++) {
513 auto &cell = get_cell(i, j);
514 auto node_map = generate_node_map(*this, cell, Mode::PROCESSOR, INT(0));
515 output_block_connectivity(cell, node_map);
516 if (parallel_size() > 1) {
517 output_nodal_communication_map(cell, node_map);
518 }
519 }
520 }
521 if (debug_level & 2) {
522 util().progress("\tEnd Nodal Communication Map Output");
523 }
524
525 if (parallel_size() > 1) {
526 for (size_t j = 0; j < JJ(); j++) {
527 for (size_t i = 0; i < II(); i++) {
528 auto &cell = get_cell(i, j);
529 output_node_map(cell, INT(0));
530 output_element_map(cell, INT(0));
531 }
532 }
533 if (debug_level & 2) {
534 util().progress("\tEnd Node/Element Map Output");
535 }
536 }
537 }
538
539 void Grid::output_nodal_coordinates(const Cell &cell)
540 {
541 int rank = cell.rank(Loc::C);
542
543 auto *nb = cell.region()->get_node_blocks()[0];
544 std::vector<double> coord_x;
545 std::vector<double> coord_y;
546 std::vector<double> coord_z;
547
548 // Are we modifying the coordinates ... scale and/or offset and/or offset_unit_cell...
549 bool mod_x = cell.m_offX != 0.0 || m_scaleFactor != 1.0 || m_offset[0] != 0.0;
550 bool mod_y = cell.m_offY != 0.0 || m_scaleFactor != 1.0 || m_offset[1] != 0.0;
551 bool mod_z = m_scaleFactor != 1.0 || m_offset[2] != 0.0;
552
553 double scale = m_scaleFactor;
554 nb->get_field_data("mesh_model_coordinates_x", coord_x);
555 if (mod_x) {
556 double offset = m_offset[0];
557 std::for_each(coord_x.begin(), coord_x.end(),
558 [&cell, scale, offset](double &d) { d = (d + cell.m_offX) * scale + offset; });
559 }
560
561 nb->get_field_data("mesh_model_coordinates_y", coord_y);
562 if (mod_y) {
563 double offset = m_offset[1];
564 std::for_each(coord_y.begin(), coord_y.end(),
565 [&cell, scale, offset](double &d) { d = (d + cell.m_offY) * scale + offset; });
566 }
567
568 nb->get_field_data("mesh_model_coordinates_z", coord_z);
569 if (mod_z) {
570 double offset = m_offset[2];
571 std::for_each(coord_z.begin(), coord_z.end(),
572 [scale, offset](double &d) { d = d * scale + offset; });
573 }
574
575 // Filter coordinates down to only "new nodes"...
576 if (m_equivalenceNodes && (cell.has_neighbor_i() || cell.has_neighbor_j())) {
577 auto mode = parallel_size() > 1 ? Mode::PROCESSOR : Mode::GLOBAL;
578 auto categorized_nodes = cell.categorize_nodes(mode);
579 size_t nn = 0;
580 for (size_t n = 0; n < categorized_nodes.size(); n++) {
581 if (categorized_nodes[n] == 0) {
582 coord_x[nn] = coord_x[n];
583 coord_y[nn] = coord_y[n];
584 coord_z[nn] = coord_z[n];
585 nn++;
586 }
587 }
588 }
589
590 int exoid = output_region(rank)->get_database()->get_file_pointer();
591 auto start = cell.m_localNodeIdOffset + 1;
592 auto count = cell.added_node_count(Mode::PROCESSOR, m_equivalenceNodes);
593 ex_put_partial_coord(exoid, start, count, Data(coord_x), Data(coord_y), Data(coord_z));
594
595 if (minimize_open_files(Minimize::UNIT)) {
596 cell.region()->get_database()->closeDatabase();
597 }
598 }
599
600 template <typename INT> void Grid::output_generated_surfaces(Cell &cell, INT /*dummy*/)
601 {
602 auto generated_ijk = get_generated_sidesets();
603 if (generated_ijk == 0) {
604 return;
605 }
606
607 int rank = cell.rank(Loc::C);
608
609 const std::array<int, 6> boundary_rank{
610 cell.rank(Loc::L), cell.rank(Loc::R), cell.rank(Loc::B), cell.rank(Loc::T), -1, -1};
611 std::array<enum Flg, 6> boundary_flag{Flg::MIN_I, Flg::MAX_I, Flg::MIN_J,
612 Flg::MAX_J, Flg::MIN_K, Flg::MAX_K};
613
614 int exoid = output_region(rank)->get_database()->get_file_pointer();
615
616 for (int face = 0; face < 6; face++) {
617 if ((generated_ijk & (unsigned)boundary_flag[face]) && boundary_rank[face] == -1) {
618 // Find surface on output mesh...
619 auto *osurf = output_region(rank)->get_sideset(generated_surface_names[face]);
620 SMART_ASSERT(osurf != nullptr);
621 const auto &oblocks = osurf->get_side_blocks();
622 SMART_ASSERT(oblocks.size() == 1)(oblocks.size());
623
624 auto &boundary = cell.unit()->boundary_blocks[face];
625 auto count = boundary.size();
626 std::vector<INT> elements;
627 std::vector<INT> faces;
628 elements.reserve(count);
629 faces.reserve(count);
630
631 for (const auto &block_faces : boundary.m_faces) {
632 const auto &block_name = block_faces.first;
633 const auto &bnd_faces = block_faces.second;
634
635 // This is the offset within this element block -- i.e., the 'element_offsetth' element in
636 // this block.
637 size_t element_offset = cell.m_localElementIdOffset[block_name];
638 // This is the offset of the elements in this element block within all of the elements in
639 // the output file.
640 size_t global_offset = output_region(rank)->get_element_block(block_name)->get_offset();
641
642 for (const auto &bface : bnd_faces) {
643 elements.push_back(bface / 10 + element_offset + global_offset);
644 faces.push_back(bface % 10 + 1);
645 }
646 }
647
648 auto id = osurf->get_property("id").get_int();
649 auto start = cell.m_localSurfaceOffset[generated_surface_names[face]];
650 ex_put_partial_set(exoid, EX_SIDE_SET, id, start + 1, count, Data(elements), Data(faces));
651 }
652 }
653 }
654
655 template <typename INT> void Grid::output_surfaces(Cell &cell, INT /*dummy*/)
656 {
657 int rank = cell.rank(Loc::C);
658 int exoid = output_region(rank)->get_database()->get_file_pointer();
659
660 // Get the surfaces on this cell...
661 const auto &surfaces = cell.region()->get_sidesets();
662 for (const auto *surface : surfaces) {
663
664 // Find corresponding surface on output mesh...
665 auto *osurf = output_region(rank)->get_sideset(surface->name());
666 SMART_ASSERT(osurf != nullptr);
667 const auto &oblocks = osurf->get_side_blocks();
668
669 std::vector<INT> elements;
670 std::vector<INT> faces;
671 elements.reserve(oblocks[0]->entity_count());
672 faces.reserve(oblocks[0]->entity_count());
673
674 const auto &blocks = surface->get_side_blocks();
675 for (const auto *block : blocks) {
676 // Get the element/face pairs for the SideBlock in this surface...
677 std::vector<INT> element_side;
678 block->get_field_data("element_side_raw", element_side);
679
680 // Now separate the element and the face into different vectors and
681 // update the element number to match the elements in this cell...
682 const auto *parent = block->parent_element_block();
683 SMART_ASSERT(parent != nullptr);
684
685 // This is the offset of the elements in the unit cell parent element block
686 size_t unit_block_offset = parent->get_offset();
687
688 // This is the offset within this element block -- i.e., the 'element_offsetth' element in
689 // this block.
690 size_t element_offset = cell.m_localElementIdOffset[parent->name()];
691 // This is the offset of the elements in this element block within all of the elements in the
692 // output file.
693 size_t global_offset = output_region(rank)->get_element_block(parent->name())->get_offset();
694
695 size_t entity_count = block->entity_count();
696 for (size_t i = 0; i < entity_count; i++) {
697 auto element = element_side[2 * i + 0];
698 auto output_element = element - unit_block_offset + element_offset + global_offset;
699 elements.push_back(output_element);
700 faces.push_back(element_side[2 * i + 1]);
701 }
702 }
703
704 auto id = osurf->get_property("id").get_int();
705 auto start = cell.m_localSurfaceOffset[osurf->name()];
706 auto count = elements.size();
707 ex_put_partial_set(exoid, EX_SIDE_SET, id, start + 1, count, Data(elements), Data(faces));
708 }
709
710 if (minimize_open_files(Minimize::UNIT)) {
711 cell.region()->get_database()->closeDatabase();
712 }
713 }
714
715 template <typename INT>
716 void Grid::output_block_connectivity(Cell &cell, const std::vector<INT> &node_map)
717 {
718 int rank = cell.rank(Loc::C);
719 if (rank >= m_startRank && rank < m_startRank + m_rankCount) {
720 int exoid = output_region(rank)->get_database()->get_file_pointer();
721
722 const auto &blocks = cell.region()->get_element_blocks();
723 std::vector<INT> connect;
724 for (const auto *block : blocks) {
725 block->get_field_data("connectivity_raw", connect);
726 for (size_t k = 0; k < connect.size(); k++) {
727 connect[k] = node_map[connect[k]];
728 }
729 auto start = cell.m_localElementIdOffset[block->name()] + 1;
730 auto count = block->entity_count();
731 auto id = block->get_property("id").get_int();
732 if (debug_level & 8) {
733 fmt::print(stderr, "Rank: {}, Cell({}, {}), Block {}, id {}, start {}, count {}\n", rank,
734 cell.m_i, cell.m_j, block->name(), id, start, count);
735 }
736 ex_put_partial_conn(exoid, EX_ELEM_BLOCK, id, start, count, Data(connect), nullptr, nullptr);
737 }
738
739 if (debug_level & 2) {
740 util().progress(fmt::format("Generated Node Map / Output Connectivity for Cell({}, {})",
741 cell.m_i, cell.m_j));
742 }
743 if (minimize_open_files(Minimize::UNIT)) {
744 cell.region()->get_database()->closeDatabase();
745 }
746 if (minimize_open_files(Minimize::OUTPUT)) {
747 output_region(rank)->get_database()->closeDatabase();
748 }
749 }
750 }
751
752 template <typename INT>
753 void Grid::output_nodal_communication_map(Cell &cell, const std::vector<INT> &node_map)
754 {
755 // The `node_map` has the processor-local node ids for all nodes on this cell.
756 // Note that the `node_map` starts at index 1 in the vector...
757 // Need to check the boundaries of the cell and determine which boundaries are on a different rank
758 // and output the nodes and processors to the communication map.
759 int rank = cell.rank(Loc::C);
760 if (rank >= m_startRank && rank < m_startRank + m_rankCount) {
761 std::vector<INT> nodes;
762 std::vector<INT> procs;
763 cell.populate_node_communication_map(node_map, nodes, procs);
764
765 int exoid = output_region(rank)->get_database()->get_file_pointer();
766
767 auto start = cell.m_communicationNodeOffset + 1;
768 auto count = cell.m_communicationNodeCount;
769
770 ex_put_partial_node_cmap(exoid, 1, start, count, Data(nodes), Data(procs), rank);
771
772 if (minimize_open_files(Minimize::OUTPUT)) {
773 output_region(rank)->get_database()->closeDatabase();
774 }
775
776 if (debug_level & 32) {
777 fmt::print(stderr, "Rank: {}, Cell({}, {}), Node Comm Map: start {}, count {}\n", rank,
778 cell.m_i, cell.m_j, start, count);
779 }
780
781 if (debug_level & 2) {
782 util().progress(
783 fmt::format("Output Nodal Communication Map for Cell({}, {})", cell.m_i, cell.m_j));
784 }
785 }
786 }
787
788 template <typename INT> void Grid::output_node_map(const Cell &cell, INT /*dummy*/)
789 {
790 int rank = cell.rank(Loc::C);
791
792 auto start = cell.m_localNodeIdOffset + 1;
793 auto count = cell.added_node_count(Mode::PROCESSOR, m_equivalenceNodes);
794
795 if (parallel_size() == 1) {
796 INT gid = cell.m_globalNodeIdOffset + 1;
797 std::vector<INT> map(count);
798 std::iota(map.begin(), map.end(), gid);
799 int exoid = output_region(rank)->get_database()->get_file_pointer();
800 ex_put_partial_id_map(exoid, EX_NODE_MAP, start, count, Data(map));
801 }
802 else {
803 auto map = generate_node_map(*this, cell, Mode::GLOBAL, INT(0));
804
805 if (rank >= m_startRank && rank < m_startRank + m_rankCount) {
806
807 // Filter nodes down to only "new nodes"...
808 if (m_equivalenceNodes && (cell.has_neighbor_i() || cell.has_neighbor_j())) {
809 auto mode = Mode::PROCESSOR;
810 auto categorized_nodes = cell.categorize_nodes(mode);
811 size_t nn = 0;
812 for (size_t n = 0; n < categorized_nodes.size(); n++) {
813 if (categorized_nodes[n] == 0) {
814 map[nn + 1] = map[n + 1];
815 nn++;
816 }
817 }
818 }
819 if (debug_level & 8) {
820 fmt::print("Cell({}, {}), start {}, count {}\n", cell.m_i, cell.m_j, start, count);
821 }
822 int exoid = output_region(rank)->get_database()->get_file_pointer();
823 ex_put_partial_id_map(exoid, EX_NODE_MAP, start, count, &map[1]);
824 if (minimize_open_files(Minimize::OUTPUT)) {
825 output_region(rank)->get_database()->closeDatabase();
826 }
827 }
828 }
829
830 if (debug_level & 2) {
831 util().progress(fmt::format("Generated Node Map for Rank {}, Cell({}, {}): start {}, count "
832 "{}\n",
833 rank, cell.m_i, cell.m_j, start, count));
834 }
835 }
836
837 template <typename INT> void Grid::output_element_map(Cell &cell, INT /*dummy*/)
838 {
839 int rank = cell.rank(Loc::C);
840 if (rank >= m_startRank && rank < m_startRank + m_rankCount) {
841 int exoid = output_region(rank)->get_database()->get_file_pointer();
842
843 const auto &output_blocks = output_region(rank)->get_element_blocks();
844
845 // This is the element block offset for the "single output file"
846 // for the block being output For example, if the total mesh has
847 // 3 blocks, with 100, 200, 100 elements, then the element block
848 // offset would be 0, 100, 300 (Ioss::ElementBlock::get_offset())
849 size_t global_id_offset = 0;
850
851 for (const auto *output_element_block : output_blocks) {
852 auto *block = cell.region()->get_element_block(output_element_block->name());
853 if (block != nullptr) {
854
855 INT gid = cell.m_globalElementIdOffset[block->name()] + 1 + global_id_offset;
856 auto element_count = block->entity_count();
857 std::vector<INT> map(element_count);
858
859 std::iota(map.begin(), map.end(), gid);
860
861 auto output_block_offset = output_element_block->get_offset();
862
863 // This cells element block ids start this far into the portion of the map for this
864 // element block
865 auto local_offset = cell.m_localElementIdOffset[block->name()];
866
867 auto start = output_block_offset + local_offset + 1;
868 ex_put_partial_id_map(exoid, EX_ELEM_MAP, start, element_count, Data(map));
869
870 if (debug_level & 8) {
871 fmt::print("Rank {}: Cell({}, {}), Block {}, start {}, element_count {}, gid {}\n", rank,
872 cell.m_i, cell.m_j, block->name(), start, element_count, gid);
873 }
874 }
875 // If we were outputting a single file, then this element
876 // block in that file would have this many elements.
877 auto global_block_element_count =
878 output_element_block->get_property("global_entity_count").get_int();
879 global_id_offset += global_block_element_count;
880 }
881 if (minimize_open_files(Minimize::OUTPUT)) {
882 output_region(rank)->get_database()->closeDatabase();
883 }
884 }
885 }
886
887 void Grid::handle_file_count()
888 {
889 // If the user has specified a 'minimize_open_files' behavior on the command line,
890 // honor that mode here. Note that we have already processed the unit cells and
891 // if the unit cell count exceeds the open file limit, then that code has already
892 // automatically set the minimize_open_files to include UNIT...
893 if (m_minimizeOpenFiles == Minimize::ALL) {
894 return;
895 }
896
897 size_t open_files = open_file_limit();
898 if (util().parallel_rank() == 0) {
899 fmt::print("\n Maximum Open File Count = {}\n", open_file_limit());
900 }
901
902 auto unit_cell_size = unit_cells().size();
903 if (minimize_open_files(Minimize::UNIT)) {
904 unit_cell_size = 1; // There will only be a single unit cell file open at a time.
905 }
906
907 if (unit_cell_size + m_rankCount > open_files) {
908 // Too many files, need to close some of them after accessing...
909 // We have already checked the unit_cell() size earlier.
910 // If user has specified 'minimize_open_files == OUTPUT', then honor that...
911 if (minimize_open_files(Minimize::OUTPUT)) {
912 return;
913 }
914
915 // Have decision on whether to keep all unit_files open
916 // and reduce open output files further, or to close
917 // unit files and have more output files open...
918 int output_open = open_files - (int)unit_cell_size;
919 if (output_open < (int)(.2 * m_rankCount)) {
920 // Close the unit files. Prefer to have output files open instead.
921 m_minimizeOpenFiles = Minimize((unsigned)m_minimizeOpenFiles | (unsigned)Minimize::UNIT);
922 unit_cell_size = 1;
923 }
924
925 // Set the rank count to the number of files left after subtracting off 'unit_cell_size'
926 // and set the 'subcycle' mode.
927 auto max_rank_count = open_files - unit_cell_size;
928 if (max_rank_count < (size_t)m_rankCount) {
929 m_rankCount = max_rank_count;
930 }
931 m_subCycle = true;
932 }
933
934 if (util().parallel_rank() == 0 && m_minimizeOpenFiles != Minimize::NONE) {
935 std::array<std::string, 4> smode{"NONE", "UNIT", "OUTPUT", "ALL"};
936 fmt::print(" Setting `minimize_open_files` mode to {}.\n", smode[(int)m_minimizeOpenFiles]);
937 }
938 }
939
940 namespace {
941 size_t handle_elements(Grid &grid, int start_rank, int rank_count)
942 {
943 // Not all unit cells have the same element blocks and the output
944 // grid will contain the union of the element blocks on each unit
945 // cell...
946 //
947 // While finalizing the cells, we will create this union for use in
948 // the output region. Would be quicker to do iteration on unit_cell
949 // map which is smaller, but for now do it here and see if becomes
950 // bottleneck.
951
952 std::map<std::string, std::unique_ptr<Ioss::ElementBlock>> output_element_blocks;
953 std::vector<std::map<std::string, size_t>> element_block_elem_count(grid.parallel_size());
954 std::map<std::string, size_t> global_element_block_elem_count;
955
956 for (size_t j = 0; j < grid.JJ(); j++) {
957 for (size_t i = 0; i < grid.II(); i++) {
958 auto &cell = grid.get_cell(i, j);
959 auto rank = cell.rank(Loc::C);
960
961 const auto &element_blocks = cell.region()->get_element_blocks();
962 for (const auto *block : element_blocks) {
963 const auto &blk = block->name();
964 cell.m_globalElementIdOffset[blk] = global_element_block_elem_count[blk];
965 cell.m_localElementIdOffset[blk] = element_block_elem_count[rank][blk];
966
967 element_block_elem_count[rank][blk] += block->entity_count();
968 global_element_block_elem_count[blk] += block->entity_count();
969
970 if (debug_level & 8) {
971 fmt::print("rank, i, j, blk, loffset, goffset, block_count: {}: {} {} {} {} {} {}\n",
972 rank, i, j, blk, cell.m_localElementIdOffset[blk],
973 cell.m_globalElementIdOffset[blk], element_block_elem_count[rank][blk]);
974 }
975
976 // Create output element block if does not exist yet...
977 if (output_element_blocks.find(blk) == output_element_blocks.end()) {
978 output_element_blocks.emplace(
979 blk, std::make_unique<Ioss::ElementBlock>(Ioss::ElementBlock(*block)));
980 }
981 }
982 }
983 }
984
985 // Calculate values needed to set the "global_entity_count" property on the output element
986 // blocks.
987 size_t global_element_count = 0;
988 std::map<const std::string, int64_t> global_block_element_count;
989 for (auto &blk : output_element_blocks) {
990 for (int rank = 0; rank < grid.parallel_size(); rank++) {
991 auto *block = blk.second.get();
992 global_block_element_count[block->name()] += element_block_elem_count[rank][block->name()];
993 global_element_count += element_block_elem_count[rank][block->name()];
994 }
995 }
996
997 // Define the element blocks in the output database...
998 for (int rank = start_rank; rank < start_rank + rank_count; rank++) {
999 for (auto &blk : output_element_blocks) {
1000 auto *block = new Ioss::ElementBlock(*blk.second);
1001 block->property_update("entity_count", element_block_elem_count[rank][block->name()]);
1002 block->property_update("global_entity_count", global_block_element_count[block->name()]);
1003 grid.output_region(rank)->property_add(
1004 Ioss::Property("global_element_count", (int64_t)global_element_count));
1005 grid.output_region(rank)->add(block);
1006 if (debug_level & 8) {
1007 fmt::print("rank, blk, element_count: {}: {} {}\n", rank, block->name(),
1008 block->entity_count());
1009 }
1010 }
1011 }
1012 return global_element_count;
1013 }
1014
1015 size_t handle_nodes(Grid &grid, int start_rank, int rank_count)
1016 {
1017 size_t global_node_count = 0;
1018 std::vector<size_t> local_node_count(grid.parallel_size());
1019
1020 for (size_t j = 0; j < grid.JJ(); j++) {
1021 for (size_t i = 0; i < grid.II(); i++) {
1022 auto &cell = grid.get_cell(i, j);
1023 auto rank = cell.rank(Loc::C);
1024 cell.m_globalNodeIdOffset = global_node_count;
1025 cell.m_localNodeIdOffset = local_node_count[rank];
1026 SMART_ASSERT(cell.region() != nullptr)(i)(j);
1027
1028 auto new_global_nodes = cell.added_node_count(Mode::GLOBAL, grid.equivalence_nodes());
1029 auto new_processor_nodes = cell.added_node_count(Mode::PROCESSOR, grid.equivalence_nodes());
1030 if (debug_level & 8) {
1031 fmt::print("rank: i, j, node_offset, added_nodes: {}: {} {} {} {} {}\n", rank, i, j,
1032 local_node_count[rank], new_global_nodes, new_processor_nodes);
1033 }
1034 local_node_count[rank] += new_processor_nodes;
1035 global_node_count += new_global_nodes;
1036 }
1037 }
1038 // Define the output database node block...
1039 for (int i = start_rank; i < start_rank + rank_count; i++) {
1040 std::string block_name = "nodeblock_1";
1041 int spatial_dimension = 3;
1042 auto *block = new Ioss::NodeBlock(grid.output_region(i)->get_database(), block_name,
1043 local_node_count[i], spatial_dimension);
1044 block->property_add(Ioss::Property("id", 1));
1045 grid.output_region(i)->add(block);
1046 grid.output_region(i)->property_add(
1047 Ioss::Property("global_node_count", (int64_t)global_node_count));
1048 }
1049 return global_node_count;
1050 }
1051
1052 void handle_communications(Grid &grid, int start_rank, int rank_count)
1053 {
1054 // Need to determine the number of nodes on the processor
1055 // boundaries for each output database and create a CommSet...
1056 std::vector<size_t> bnode(grid.parallel_size());
1057 for (size_t j = 0; j < grid.JJ(); j++) {
1058 for (size_t i = 0; i < grid.II(); i++) {
1059 auto &cell = grid.get_cell(i, j);
1060 auto rank = cell.rank(Loc::C);
1061 cell.m_communicationNodeOffset = bnode[rank];
1062 auto bnd = cell.processor_boundary_node_count();
1063 bnode[rank] += bnd;
1064 if (debug_level & 32) {
1065 fmt::print("rank: {}, Cell({}, {}): Boundary Count = {}, Total = {}\n", rank, i, j, bnd,
1066 bnode[rank]);
1067 }
1068 }
1069 }
1070
1071 // Now add communication sets to all output databases...
1072 for (int rank = start_rank; rank < start_rank + rank_count; rank++) {
1073 auto *cs = new Ioss::CommSet(grid.output_region(rank)->get_database(), "commset_node", "node",
1074 bnode[rank]);
1075 grid.output_region(rank)->add(cs);
1076 }
1077 }
1078
1079 void handle_surfaces(Grid &grid, int start_rank, int rank_count)
1080 {
1081 // Not all unit cells have the same surfaces and the output
1082 // grid will contain the union of the surfaces on each unit
1083 // cell that has a surface on the boundary of the output mesh.
1084 //
1085 // While finalizing the cells, we will create this union for use in
1086 // the output region. Would be quicker to do iteration on unit_cell
1087 // map which is smaller, but for now do it here and see if becomes
1088 // bottleneck.
1089 std::map<std::string, std::unique_ptr<Ioss::SideSet>> output_surfaces;
1090 std::map<std::string, size_t> global_surface_face_count;
1091
1092 // Per rank...
1093 std::vector<std::map<std::string, size_t>> surface_face_count(grid.parallel_size());
1094 std::vector<std::map<std::string, size_t>> local_surface_offset(grid.parallel_size());
1095
1096 for (size_t j = 0; j < grid.JJ(); j++) {
1097 for (size_t i = 0; i < grid.II(); i++) {
1098 auto &cell = grid.get_cell(i, j);
1099 auto rank = cell.rank(Loc::C);
1100
1101 const auto &surfaces = cell.region()->get_sidesets();
1102 for (const auto *surface : surfaces) {
1103 const auto &surf = surface->name();
1104 cell.m_localSurfaceOffset[surf] = local_surface_offset[rank][surf];
1105
1106 const auto &blocks = surface->get_side_blocks();
1107 for (const auto *blk : blocks) {
1108 surface_face_count[rank][surf] += blk->entity_count();
1109 global_surface_face_count[surf] += blk->entity_count();
1110 local_surface_offset[rank][surf] += blk->entity_count();
1111 }
1112
1113 // Create output surface if does not exist yet...
1114 if (output_surfaces.find(surf) == output_surfaces.end()) {
1115 output_surfaces.emplace(surf, std::make_unique<Ioss::SideSet>(Ioss::SideSet(*surface)));
1116 }
1117 }
1118 }
1119 }
1120
1121 // Define the surfaces in the output database...
1122 for (int rank = start_rank; rank < start_rank + rank_count; rank++) {
1123 for (auto &surf : output_surfaces) {
1124 auto *surface =
1125 new Ioss::SideSet(grid.output_region(rank)->get_database(), surf.second->name());
1126 auto *block =
1127 new Ioss::SideBlock(grid.output_region(rank)->get_database(), surf.second->name(),
1128 "quad4", "hex8", surface_face_count[rank][surface->name()]);
1129 surface->add(block);
1130 block->property_update("global_entity_count", global_surface_face_count[surface->name()]);
1131 surface->property_update("global_entity_count", global_surface_face_count[surface->name()]);
1132 block->property_update("distribution_factor_count", 0);
1133 grid.output_region(rank)->add(surface);
1134 }
1135 }
1136 }
1137
1138 void generate_surfaces(Grid &grid, int start_rank, int rank_count)
1139 {
1140 // The user may have requested that sidesets on one or more
1141 // of the models boundary surfaces (min i,j,k and/or max i,j,k)
1142 // be automatically generated (they don't exist on the input
1143 // unit cells). Handle those sidesets here.
1144
1145 auto generated_ijk = grid.get_generated_sidesets();
1146 if (generated_ijk == 0) {
1147 return;
1148 }
1149
1150 std::array<size_t, 6> global_surface_face_count{};
1151
1152 // Per rank...
1153 std::vector<std::array<size_t, 6>> surface_face_count(grid.parallel_size());
1154 std::vector<std::array<size_t, 6>> local_surface_offset(grid.parallel_size());
1155
1156 std::array<enum Flg, 6> boundary_flag{Flg::MIN_I, Flg::MAX_I, Flg::MIN_J,
1157 Flg::MAX_J, Flg::MIN_K, Flg::MAX_K};
1158
1159 for (size_t j = 0; j < grid.JJ(); j++) {
1160 for (size_t i = 0; i < grid.II(); i++) {
1161 auto &cell = grid.get_cell(i, j);
1162 auto rank = cell.rank(Loc::C);
1163 const std::array<int, 6> boundary_rank{
1164 cell.rank(Loc::L), cell.rank(Loc::R), cell.rank(Loc::B), cell.rank(Loc::T), -1, -1};
1165
1166 for (int face = 0; face < 6; face++) {
1167 if ((generated_ijk & (unsigned)boundary_flag[face]) && boundary_rank[face] == -1) {
1168 cell.m_localSurfaceOffset[grid.generated_surface_names[face]] =
1169 local_surface_offset[rank][face];
1170
1171 auto count = cell.unit()->boundary_blocks[face].size();
1172 global_surface_face_count[face] += count;
1173 surface_face_count[rank][face] += count;
1174 local_surface_offset[rank][face] += count;
1175 }
1176 }
1177 }
1178 }
1179
1180 // Define the surfaces in the output database...
1181 for (int rank = start_rank; rank < start_rank + rank_count; rank++) {
1182 for (size_t i = 0; i < 6; i++) {
1183 if (global_surface_face_count[i] > 0) {
1184 auto *surface = new Ioss::SideSet(grid.output_region(rank)->get_database(),
1185 grid.generated_surface_names[i]);
1186 auto *block = new Ioss::SideBlock(grid.output_region(rank)->get_database(),
1187 grid.generated_surface_names[i], "quad4", "hex8",
1188 surface_face_count[rank][i]);
1189 surface->property_update("global_entity_count", global_surface_face_count[i]);
1190 block->property_update("global_entity_count", global_surface_face_count[i]);
1191 block->property_update("distribution_factor_count", 0);
1192 surface->add(block);
1193 grid.output_region(rank)->add(surface);
1194 }
1195 }
1196 }
1197 }
1198
1199 template <typename INT>
1200 std::vector<INT> generate_node_map(Grid &grid, const Cell &cell, Mode mode, INT /*dummy*/)
1201 {
1202 // Generate a "map" from nodes in the input connectivity to the
1203 // output connectivity in global nodes. If no neighbors, then
1204 // this would just be adding `cell.m_globalNodeIdOffset` to each
1205 // connectivity entry.
1206
1207 std::vector<INT> map = cell.generate_node_map(mode, grid.equivalence_nodes(), INT(0));
1208 if (debug_level & 8) {
1209 fmt::print("Cell({},{}) PROCESSOR MAP: {}\n", cell.m_i, cell.m_j, fmt::join(map, " "));
1210 }
1211
1212 // Now that we have the node map for this cell, we need to save
1213 // the mappings for the max_I and max_J faces and max_I-max_J edge
1214 // for use by later neighbors... Check whether cell has neighbors
1215 // on max_I or max_J faces...
1216 if (grid.equivalence_nodes()) {
1217 if ((mode == Mode::GLOBAL ||
1218 (mode == Mode::PROCESSOR && cell.rank(Loc::C) == cell.rank(Loc::R))) &&
1219 (cell.m_i + 1 < grid.II())) {
1220 const auto &neighbor = grid.get_cell(cell.m_i + 1, cell.m_j);
1221 cell.populate_neighbor(Loc::L, map, neighbor);
1222 }
1223
1224 if ((mode == Mode::GLOBAL ||
1225 (mode == Mode::PROCESSOR && cell.rank(Loc::C) == cell.rank(Loc::T))) &&
1226 (cell.m_j + 1 < grid.JJ())) {
1227 const auto &neighbor = grid.get_cell(cell.m_i, cell.m_j + 1);
1228 cell.populate_neighbor(Loc::B, map, neighbor);
1229 }
1230
1231 if (mode == Mode::PROCESSOR) {
1232 if (cell.processor_boundary(Loc::L) && (cell.rank(Loc::TL) == cell.rank(Loc::C))) {
1233 const auto &tl_corner = grid.get_cell(cell.m_i - 1, cell.m_j + 1);
1234 cell.populate_neighbor(Loc::BR, map, tl_corner);
1235 }
1236 // Now the other "corner case"
1237 if (cell.processor_boundary(Loc::R) && (cell.rank(Loc::TR) == cell.rank(Loc::C))) {
1238 const auto &tr_corner = grid.get_cell(cell.m_i + 1, cell.m_j + 1);
1239 cell.populate_neighbor(Loc::BL, map, tr_corner);
1240 }
1241 }
1242 }
1243 return map;
1244 }
1245
1246 Ioss::PropertyManager parse_properties(SystemInterface &interFace, int int_size)
1247 {
1248 Ioss::PropertyManager properties;
1249 if (int_size == 8) {
1250 properties.add(Ioss::Property("INTEGER_SIZE_DB", 8));
1251 properties.add(Ioss::Property("INTEGER_SIZE_API", 8));
1252 }
1253
1254 if (interFace.use_netcdf4()) {
1255 properties.add(Ioss::Property("FILE_TYPE", "netcdf4"));
1256 }
1257
1258 if (interFace.use_netcdf5()) {
1259 properties.add(Ioss::Property("FILE_TYPE", "netcdf5"));
1260 }
1261
1262 if (interFace.compression_level() > 0 || interFace.szip()) {
1263 properties.add(Ioss::Property("FILE_TYPE", "netcdf4"));
1264 properties.add(Ioss::Property("COMPRESSION_LEVEL", interFace.compression_level()));
1265 properties.add(Ioss::Property("COMPRESSION_SHUFFLE", true));
1266 if (interFace.szip()) {
1267 properties.add(Ioss::Property("COMPRESSION_METHOD", "szip"));
1268 }
1269 else if (interFace.zlib()) {
1270 properties.add(Ioss::Property("COMPRESSION_METHOD", "zlib"));
1271 }
1272 }
1273 return properties;
1274 }
1275
1276 std::shared_ptr<Ioss::Region> create_input_region(const std::string &key, std::string filename,
1277 bool ints_32_bits)
1278 {
1279 // Check that 'filename' does not contain a starting/ending double quote...
1280 filename.erase(remove(filename.begin(), filename.end(), '\"'), filename.end());
1281 Ioss::DatabaseIO *dbi = Ioss::IOFactory::create("exodus", filename, Ioss::READ_RESTART,
1282 Ioss::ParallelUtils::comm_self());
1283 if (dbi == nullptr || !dbi->ok(true)) {
1284 std::exit(EXIT_FAILURE);
1285 }
1286
1287 dbi->set_surface_split_type(Ioss::SPLIT_BY_DONT_SPLIT);
1288 if (ints_32_bits) {
1289 dbi->set_int_byte_size_api(Ioss::USE_INT32_API);
1290 }
1291 else {
1292 dbi->set_int_byte_size_api(Ioss::USE_INT64_API);
1293 }
1294
1295 // Splitting surfaces by element block makes it easier to transform the input
1296 // element id into the output element ids.
1297 dbi->set_surface_split_type(Ioss::SPLIT_BY_ELEMENT_BLOCK);
1298
1299 // Generate a name for the region based on the key...
1300 std::string name = "Region_" + key;
1301 // NOTE: region owns database pointer at this time...
1302 return std::make_shared<Ioss::Region>(dbi, name);
1303 }
1304
1305 void output_summary(Ioss::Region *region, std::ostream &strm)
1306 {
1307 int64_t nodes = region->get_property("node_count").get_int();
1308 int64_t elements = region->get_property("element_count").get_int();
1309
1310 fmt::print(strm, " Database: {}\tNodes = {} \tElements = {}\n",
1311 region->get_database()->get_filename(), fmt::group_digits(nodes),
1312 fmt::group_digits(elements));
1313 }
1314
1315 } // namespace