Debian Codegraph @

packages/seacas/applications/slice/Slice.C

raw
1 // Copyright(C) 1999-2024 National Technology & Engineering Solutions
2 // of Sandia, LLC (NTESS). Under the terms of Contract DE-NA0003525 with
3 // NTESS, the U.S. Government retains certain rights in this software.
4 //
5 // See packages/seacas/LICENSE for details
6
7 #include <SL_Decompose.h>
8 #include <SL_SystemInterface.h>
9 #include <SL_Version.h>
10
11 #include <Ioss_CodeTypes.h>
12 #include <Ioss_CopyDatabase.h>
13 #include <Ioss_DatabaseIO.h>
14 #include <Ioss_DecompositionUtils.h>
15 #include <Ioss_FileInfo.h>
16 #include <Ioss_MemoryUtils.h>
17 #include <Ioss_MeshCopyOptions.h>
18 #include <Ioss_Region.h>
19 #include <Ioss_SubSystem.h>
20 #include <Ioss_SurfaceSplit.h>
21 #include <Ioss_Utils.h>
22 #include <cassert>
23 #include <fmt/color.h>
24 #include <fmt/format.h>
25 #include <fmt/ostream.h>
26 #include <init/Ionit_Initializer.h>
27
28 #include <exodusII.h>
29
30 #include <algorithm>
31 #include <array>
32 #include <chrono>
33 #include <cstddef>
34 #include <cstdlib>
35 #include <cstring>
36 #include <map>
37 #include <numeric>
38 #include <string>
39 #include <vector>
40
41 #include <sys/types.h>
42
43 #ifdef SEACAS_HAVE_MPI
44 #include <mpi.h>
45 #endif
46
47 // ========================================================================
48 // TODO(gdsjaar):
49 // * Sideset distribution factors
50 // * Variables
51 // * All entity types
52 // * More efficient border-node-processor communication map.
53 // ========================================================================
54
55 extern double seacas_timer();
56 int debug_level = 0;
57
58 // size_t partial_count = 1'00'000;
59 size_t partial_count = 1'000'000'000;
60
61 void progress(const std::string &output)
62 {
63 static auto start = std::chrono::steady_clock::now();
64
65 if ((debug_level & 1) != 0) {
66 auto now = std::chrono::steady_clock::now();
67 std::chrono::duration<double> diff = now - start;
68 fmt::print(stderr, " [{:.2f} - {}]\t{}\n", diff.count(),
69 fmt::group_digits(Ioss::MemoryUtils::get_memory_info()), output);
70 }
71 }
72
73 namespace {
74 void proc_progress(int p, int proc_count)
75 {
76 if (((debug_level & 8) != 0) && ((proc_count <= 20) || ((p + 1) % (proc_count / 20) == 0))) {
77 progress("\t\tProcessor " + std::to_string(p + 1));
78 }
79 }
80
81 Ioss::PropertyManager set_properties(const SystemInterface &interFace)
82 {
83 Ioss::PropertyManager properties;
84 if (interFace.netcdf4_) {
85 properties.add(Ioss::Property("FILE_TYPE", "netcdf4"));
86 }
87
88 if (interFace.netcdf5_) {
89 properties.add(Ioss::Property("FILE_TYPE", "netcdf5"));
90 }
91
92 if (interFace.compressionLevel_ > 0 || interFace.shuffle_ || interFace.szip_ ||
93 interFace.zlib_ || interFace.zstd_) {
94 properties.add(Ioss::Property("FILE_TYPE", "netcdf4"));
95 properties.add(Ioss::Property("COMPRESSION_LEVEL", interFace.compressionLevel_));
96 properties.add(Ioss::Property("COMPRESSION_SHUFFLE", static_cast<int>(interFace.shuffle_)));
97
98 if (interFace.zlib_) {
99 properties.add(Ioss::Property("COMPRESSION_METHOD", "zlib"));
100 }
101 else if (interFace.szip_) {
102 properties.add(Ioss::Property("COMPRESSION_METHOD", "szip"));
103 }
104 else if (interFace.zstd_) {
105 properties.add(Ioss::Property("COMPRESSION_METHOD", "zstd"));
106 }
107 else if (interFace.bz2_) {
108 properties.add(Ioss::Property("COMPRESSION_METHOD", "bzip2"));
109 }
110
111 if (interFace.quantizeNSD_ > 0) {
112 properties.add(Ioss::Property("COMPRESSION_QUANTIZE_NSD", interFace.quantizeNSD_));
113 }
114 }
115
116 if (interFace.ints64Bit_) {
117 properties.add(Ioss::Property("INTEGER_SIZE_DB", 8));
118 properties.add(Ioss::Property("INTEGER_SIZE_API", 8));
119 }
120 return properties;
121 }
122
123 // Add the chain maps for file-per-rank output...
124 template <typename INT>
125 void output_chain_maps(std::vector<Ioss::Region *> &proc_region, const Ioss::chain_t<INT> &chains,
126 const std::vector<int> &elem_to_proc, size_t proc_begin, size_t proc_size,
127 INT /* dummy */)
128 {
129 progress(__func__);
130
131 // Build the proc-local chain map and then output it on a block-by-block basis...
132 size_t proc_count = proc_region.size();
133 std::vector<std::vector<INT>> map(proc_count);
134
135 size_t global_element_count = elem_to_proc.size();
136 for (size_t j = 0; j < global_element_count; j++) {
137 size_t p = elem_to_proc[j];
138 if (p >= proc_begin && p < proc_begin + proc_size) {
139 auto &chain_entry = chains[j];
140 // TODO: Map this from global to local element number...
141 size_t loc_elem =
142 chain_entry.element > 0
143 ? proc_region[p]->get_database()->element_global_to_local(chain_entry.element)
144 : 0;
145 map[p].push_back(loc_elem);
146 map[p].push_back(chain_entry.link);
147 }
148 }
149
150 // Now iterate through the blocks on each rank and output that portion of the map...
151 size_t block_count = proc_region[0]->get_property("element_block_count").get_int();
152
153 for (size_t p = proc_begin; p < proc_begin + proc_size; p++) {
154 const auto &proc_ebs = proc_region[p]->get_element_blocks();
155 size_t offset = 0;
156 for (size_t b = 0; b < block_count; b++) {
157 size_t proc_element_count = proc_ebs[b]->entity_count();
158 proc_ebs[b]->put_field_data("chain", &map[p][offset], proc_element_count * 2 * sizeof(INT));
159 offset += proc_element_count;
160 }
161 }
162 }
163
164 void add_chain_maps(Ioss::Region &region)
165 {
166 ex_put_map_param(region.get_database()->get_file_pointer(), 0, 2);
167 ex_put_name(region.get_database()->get_file_pointer(), EX_ELEM_MAP, 1, "chain:root_element_id");
168 ex_put_name(region.get_database()->get_file_pointer(), EX_ELEM_MAP, 2, "chain:depth_from_root");
169
170 // The chain / line data will be stored as an element map...
171 const auto &blocks = region.get_element_blocks();
172 for (const auto &block : blocks) {
173 auto field =
174 Ioss::Field("chain", region.field_int_type(), "Real[2]", Ioss::Field::MAP).set_index(1);
175 block->field_add(std::move(field));
176 }
177 }
178
179 void add_decomp_map(Ioss::Region &region, const std::string &decomp_variable_name,
180 bool add_chain_info)
181 {
182 ex_opts(EX_VERBOSE);
183 if (add_chain_info) {
184 ex_put_map_param(region.get_database()->get_file_pointer(), 0, 3);
185 ex_put_name(region.get_database()->get_file_pointer(), EX_ELEM_MAP, 1,
186 decomp_variable_name.c_str());
187 ex_put_name(region.get_database()->get_file_pointer(), EX_ELEM_MAP, 2,
188 "chain:root_element_id");
189 ex_put_name(region.get_database()->get_file_pointer(), EX_ELEM_MAP, 3,
190 "chain:depth_from_root");
191 }
192 else {
193 ex_put_map_param(region.get_database()->get_file_pointer(), 0, 1);
194 ex_put_name(region.get_database()->get_file_pointer(), EX_ELEM_MAP, 1,
195 decomp_variable_name.c_str());
196 }
197
198 // The chain / line data will be stored as an element map...
199 const auto &blocks = region.get_element_blocks();
200 for (const auto &block : blocks) {
201 auto field =
202 Ioss::Field(decomp_variable_name, Ioss::Field::INT32, IOSS_SCALAR(), Ioss::Field::MAP)
203 .set_index(1);
204 block->field_add(std::move(field));
205 if (add_chain_info) {
206 auto ch_field =
207 Ioss::Field("chain", region.field_int_type(), "Real[2]", Ioss::Field::MAP).set_index(2);
208 block->field_add(std::move(ch_field));
209 }
210 }
211 }
212
213 template <typename INT>
214 void output_decomp_map(Ioss::Region &region, const std::vector<int> &elem_to_proc,
215 const Ioss::chain_t<INT> &chains, const std::string &decomp_variable_name,
216 bool add_chain_info)
217 {
218 const auto &blocks = region.get_element_blocks();
219 size_t offset = 0;
220
221 // Convert is outputting to 64-bit database...
222 std::vector<INT> elem_to_proc64;
223 if (sizeof(INT) == 8) {
224 elem_to_proc64.reserve(elem_to_proc.size());
225 for (const auto &etp : elem_to_proc) {
226 elem_to_proc64.push_back(etp);
227 }
228 }
229 for (const auto &block : blocks) {
230 size_t num_elem = block->entity_count();
231 if (sizeof(INT) == 8) {
232 block->put_field_data(decomp_variable_name, (void *)&elem_to_proc64[offset], -1);
233 }
234 else {
235 block->put_field_data(decomp_variable_name, (void *)&elem_to_proc[offset], -1);
236 }
237 if (add_chain_info) {
238 std::vector<INT> chain;
239 chain.reserve(num_elem * 2);
240 for (size_t i = 0; i < num_elem; i++) {
241 auto &chain_entry = chains[i + offset];
242 chain.push_back(chain_entry.element);
243 chain.push_back(chain_entry.link);
244 }
245 block->put_field_data("chain", chain);
246 }
247 offset += num_elem;
248 }
249 }
250
251 void add_decomp_field(Ioss::Region &region, const std::string &decomp_variable_name,
252 bool add_chain_info)
253 {
254 region.begin_mode(Ioss::STATE_DEFINE_TRANSIENT);
255
256 const auto &blocks = region.get_element_blocks();
257 for (const auto &block : blocks) {
258 block->field_add(Ioss::Field(decomp_variable_name, region.field_int_type(), IOSS_SCALAR(),
259 Ioss::Field::TRANSIENT));
260 if (add_chain_info) {
261 block->field_add(
262 Ioss::Field("chain", region.field_int_type(), "Real[2]", Ioss::Field::TRANSIENT));
263 }
264 }
265 region.end_mode(Ioss::STATE_DEFINE_TRANSIENT);
266 }
267
268 template <typename INT>
269 void output_decomp_field(Ioss::Region &region, const std::vector<int> &elem_to_proc,
270 const Ioss::chain_t<INT> &chains,
271 const std::string &decomp_variable_name, bool add_chain_info)
272 {
273 region.begin_mode(Ioss::STATE_TRANSIENT);
274
275 auto step = region.add_state(0.0);
276 region.begin_state(step);
277
278 output_decomp_map(region, elem_to_proc, chains, decomp_variable_name, add_chain_info);
279
280 region.end_state(step);
281 region.end_mode(Ioss::STATE_TRANSIENT);
282 }
283
284 template <typename INT>
285 void populate_proc_node(size_t count, size_t offset, size_t element_nodes,
286 const std::vector<int> &elem_to_proc, const std::vector<INT> &glob_conn,
287 std::vector<std::vector<int>> &proc_node,
288 std::vector<size_t> &on_proc_count)
289 {
290 // Determine which processor(s) each node is present on.
291 // Also count number of nodes on each processor.
292 size_t el = 0;
293 for (size_t j = 0; j < count; j++) {
294 auto p = elem_to_proc[offset + j];
295 for (size_t k = 0; k < element_nodes; k++) {
296 INT node = glob_conn[el++] - 1;
297 bool exists = std::find(std::begin(proc_node[node]), std::end(proc_node[node]), p) !=
298 std::end(proc_node[node]);
299 if (!exists) {
300 proc_node[node].push_back(p);
301 on_proc_count[p]++;
302 }
303 }
304 }
305 }
306
307 void filename_substitution(std::string &filename, const SystemInterface &interFace);
308
309 template <typename INT>
310 void slice(Ioss::Region &region, const std::string &nemfile, SystemInterface &interFace,
311 INT dummy);
312
313 template <typename INT> bool is_sequential(const std::vector<INT> &map)
314 {
315 progress(__func__);
316 for (size_t i = 0; i < map.size(); i++) {
317 if (map[i] != i + 1) {
318 return false;
319 }
320 }
321 return true;
322 }
323
324 } // namespace
325 // ========================================================================
326
327 int main(int argc, char *argv[])
328 {
329 #ifdef SEACAS_HAVE_MPI
330 MPI_Init(&argc, &argv);
331 #endif
332
333 double begin = seacas_timer();
334
335 Ioss::Init::Initializer io;
336 SystemInterface::show_version();
337
338 SystemInterface interFace;
339 bool ok = interFace.parse_options(argc, argv);
340 if (!ok) {
341 fmt::print(stderr, "\nERROR: Problem parsing command line options.\n\n");
342 exit(EXIT_FAILURE);
343 }
344
345 std::string nem_file = interFace.nemesisFile_;
346 std::string path = interFace.output_path();
347 if (!path.empty()) {
348 filename_substitution(path, interFace);
349
350 // See if specified path exists.
351 Ioss::FileInfo output_path(path);
352 if (!output_path.exists()) {
353 // Try to create the directory...
354 Ioss::FileInfo::create_path(path);
355 }
356 else if (!output_path.is_dir()) {
357 fmt::print(stderr, "ERROR: Path '{}' is not a directory.\n", path);
358 exit(EXIT_FAILURE);
359 }
360
361 // See if the nem_file already has a path prepended to the
362 // filename and if so, extract the basename.
363 Ioss::FileInfo nemesis(nem_file);
364 std::string sep = "/";
365 if (path[path.length() - 1] == '/') {
366 sep = "";
367 }
368 nem_file = path + sep + nemesis.tailname();
369 }
370
371 if (interFace.outputDecompMap_ || interFace.outputDecompField_) {
372 // Then not creating split files, just adding map or field to a single output file
373 // Need to check that not overwriting input file...
374 if (interFace.inputFile_ == nem_file) {
375 nem_file += "-decomp";
376 }
377 }
378
379 fmt::print(stderr, "\nInput: '{}'\n", interFace.inputFile_);
380 fmt::print(stderr, "Output: '{}'\n", nem_file);
381
382 debug_level = interFace.debug();
383 partial_count = interFace.partial();
384
385 //========================================================================
386 // INPUT ...
387 // NOTE: The "READ_RESTART" mode ensures that the node and element ids will be mapped.
388 //========================================================================
389 Ioss::DatabaseIO *dbi =
390 Ioss::IOFactory::create(interFace.inputFormat_, interFace.inputFile_, Ioss::READ_RESTART,
391 Ioss::ParallelUtils::comm_world());
392 if (dbi == nullptr || !dbi->ok(true)) {
393 std::exit(EXIT_FAILURE);
394 }
395
396 if (interFace.ints64Bit_) {
397 dbi->set_int_byte_size_api(Ioss::USE_INT64_API);
398 }
399
400 dbi->set_surface_split_type(Ioss::SPLIT_BY_DONT_SPLIT);
401 dbi->set_field_separator(0);
402
403 try {
404 // NOTE: 'region' owns 'db' pointer at this time...
405 Ioss::Region region(dbi, "region_1");
406 region.output_summary(std::cerr, true);
407
408 if (dbi->int_byte_size_api() == 4) {
409 progress("4-byte slice");
410 slice(region, nem_file, interFace, 1);
411 }
412 else {
413 progress("8-byte slice");
414 slice(region, nem_file, interFace, static_cast<int64_t>(1));
415 }
416 }
417 catch (std::exception &e) {
418 fmt::print(stderr, "\n{}\n\nSlice terminated due to exception\n", e.what());
419 exit(EXIT_FAILURE);
420 }
421
422 #ifdef SEACAS_HAVE_MPI
423 MPI_Comm parentcomm;
424 MPI_Comm_get_parent(&parentcomm);
425 if (parentcomm != MPI_COMM_NULL) {
426 int istatus = EXIT_SUCCESS;
427 MPI_Send(&istatus, 1, MPI_INT, 0, 0, parentcomm);
428 }
429 MPI_Finalize();
430 #endif
431 fmt::print(stderr, "\nHigh-Water Memory Use: {} bytes\n",
432 fmt::group_digits(Ioss::MemoryUtils::get_hwm_memory_info()));
433 fmt::print(stderr, "Total execution time = {:.5}\n", seacas_timer() - begin);
434 fmt::print(stderr, "\nSlice execution successful.\n");
435 return EXIT_SUCCESS;
436 }
437
438 namespace {
439
440 template <typename INT>
441 void get_sidesets(const Ioss::Region &region, std::vector<Ioss::Region *> &proc_region,
442 const std::vector<int> &elem_to_proc, INT /*dummy*/)
443 {
444 progress(__func__);
445 // This routine reads the sidesets in the global database;
446 // and defines corresponding sidesets on each processor...
447 size_t proc_count = proc_region.size();
448
449 const auto &ss = region.get_sidesets();
450 size_t set_count = ss.size();
451
452 for (size_t s = 0; s < set_count; s++) {
453 auto *gss = ss[s];
454 const auto &ss_name = gss->name();
455
456 std::vector<Ioss::SideSet *> sset(proc_count);
457 for (size_t p = 0; p < proc_count; p++) {
458 sset[p] = new Ioss::SideSet(proc_region[p]->get_database(), ss_name);
459 proc_region[p]->add(sset[p]);
460 }
461
462 const auto &side_blocks = gss->get_side_blocks();
463 for (const auto &gsb : side_blocks) {
464 std::vector<INT> ss_elems;
465 gsb->get_field_data("element_side_raw", ss_elems);
466
467 std::vector<INT> pss(proc_count);
468 for (size_t i = 0; i < ss_elems.size(); i += 2 /* elem,side pairs */) {
469 int64_t elem = ss_elems[i] - 1;
470 int p = elem_to_proc[elem];
471 pss[p]++;
472 }
473
474 const auto &name = gsb->name();
475 const auto &side_type = gsb->topology()->name();
476 const auto &elem_type = gsb->parent_element_topology()->name();
477
478 for (size_t p = 0; p < proc_count; p++) {
479 auto *side_block = new Ioss::SideBlock(proc_region[p]->get_database(), name, side_type,
480 elem_type, pss[p]);
481 sset[p]->add(side_block);
482 }
483 }
484 }
485 }
486
487 template <typename INT>
488 void output_sidesets(const Ioss::Region &region, std::vector<Ioss::Region *> &proc_region,
489 const std::vector<int> &elem_to_proc, size_t proc_begin, size_t proc_size,
490 INT /*dummy*/)
491 {
492 progress(__func__);
493 // This routine reads the sidesets in the global database;
494 // and outputs the sidesets on each processor...
495 size_t proc_count = proc_region.size();
496
497 const auto &ss = region.get_sidesets();
498 size_t set_count = ss.size();
499
500 for (size_t s = 0; s < set_count; s++) {
501 if (debug_level & 4) {
502 progress("\tSideset " + std::to_string(s + 1));
503 }
504 Ioss::SideSet *gss = ss[s];
505 const auto &ss_name = gss->name();
506
507 std::vector<Ioss::SideSet *> proc_ss(proc_count);
508 for (size_t p = proc_begin; p < proc_begin + proc_size; p++) {
509 proc_ss[p] = proc_region[p]->get_sideset(ss_name);
510 }
511
512 const auto &side_blocks = gss->get_side_blocks();
513 for (const auto &gsb : side_blocks) {
514 const auto &sb_name = gsb->name();
515
516 std::vector<Ioss::SideBlock *> proc_sb(proc_count);
517 std::vector<std::vector<INT>> psb_elems(proc_count);
518 for (size_t p = proc_begin; p < proc_begin + proc_size; p++) {
519 proc_sb[p] = proc_ss[p]->get_side_block(sb_name);
520 size_t elem_count = proc_sb[p]->entity_count();
521 psb_elems[p].reserve(elem_count * 2);
522 }
523
524 std::vector<INT> ss_elems;
525 gsb->get_field_data("element_side_raw", ss_elems);
526
527 for (size_t i = 0; i < ss_elems.size(); i += 2 /* elem,side pairs */) {
528 int64_t elem = ss_elems[i] - 1;
529 int p = elem_to_proc[elem];
530 psb_elems[p].push_back(elem + 1);
531 psb_elems[p].push_back(ss_elems[i + 1]);
532 }
533
534 for (size_t p = proc_begin; p < proc_begin + proc_size; p++) {
535 Ioss::SideBlock *psb = proc_sb[p];
536 psb->put_field_data("element_side", psb_elems[p]);
537 proc_progress(p, proc_count);
538 }
539 }
540 }
541 if (set_count > 0) {
542 static bool output = false;
543 if (!output) {
544 fmt::print(stderr, "WARNING: Sideset distribution factors not yet handled correctly.\n");
545 output = true;
546 }
547 }
548 }
549
550 template <typename INT>
551 void output_communication_map(const Ioss::Region &global_region,
552 std::vector<Ioss::Region *> &proc_region,
553 const std::vector<int> &node_to_proc,
554 const std::vector<INT> &node_to_proc_pointer, size_t proc_begin,
555 size_t proc_size)
556 {
557 progress(__func__);
558
559 std::vector<std::vector<INT>> border_node_proc_map(proc_size);
560
561 INT global_node_count = global_region.get_property("node_count").get_int();
562 // Iterate all nodes and count the number of processors it is on:
563 for (INT i = 0; i < global_node_count; i++) {
564 size_t node_proc_count = node_to_proc_pointer[i + 1] - node_to_proc_pointer[i];
565 if (node_proc_count > 1) {
566 // Get the <node,proc> pairs for all border nodes on this processor...
567 // Not efficient at this time...
568 size_t beg = node_to_proc_pointer[i];
569 size_t end = node_to_proc_pointer[i + 1];
570 for (size_t j = beg; j < end; j++) {
571 size_t node = i + 1;
572 size_t proc = node_to_proc[j];
573 for (size_t k = beg; k < end; k++) {
574 if (j == k) {
575 continue;
576 }
577 size_t p = node_to_proc[k];
578 if (p >= proc_begin && p < proc_begin + proc_size) {
579 border_node_proc_map[p - proc_begin].push_back(node);
580 border_node_proc_map[p - proc_begin].push_back(proc);
581 }
582 }
583 }
584 }
585 }
586
587 progress("border_node_proc_map fully populated");
588 size_t proc_count = proc_region.size();
589 for (size_t p = proc_begin; p < proc_begin + proc_size; p++) {
590 const auto &commset = proc_region[p]->get_commsets()[0];
591 commset->put_field_data("entity_processor", border_node_proc_map[p - proc_begin]);
592 border_node_proc_map[p - proc_begin].clear();
593 proc_progress(p, proc_count);
594 }
595 }
596
597 template <typename INT>
598 void define_communication_data(const Ioss::Region &global_region,
599 std::vector<Ioss::Region *> &proc_region,
600 const std::vector<int> &node_to_proc,
601 const std::vector<INT> &node_to_proc_pointer)
602 {
603 progress(__func__);
604 // This routine categorizes the nodes on a processor as interior
605 // or border.
606 // TODO(gdsjaar): Categorize elements also. For now, all treated as
607 // interior which works for sierra-based applications
608
609 // The node_to_proc_pointer has information about the number of
610 // processors that a node is shared with. If the count is 1, then
611 // the node is interior; otherwise, it is border.
612
613 // Allocates:
614 // * interior_nodes INT size - #interior nodes
615 // * border_nodes_proc_map INT size - (proc-node) pair for each border node
616
617 INT global_node_count = global_region.get_property("node_count").get_int();
618 size_t proc_count = proc_region.size();
619 std::vector<INT> interior_nodes(proc_count);
620 std::vector<INT> border_nodes(proc_count);
621
622 // Iterate all nodes and count the number of processors it is on:
623 for (INT i = 0; i < global_node_count; i++) {
624 size_t node_proc_count = node_to_proc_pointer[i + 1] - node_to_proc_pointer[i];
625 if (node_proc_count == 1) {
626 size_t proc = node_to_proc[node_to_proc_pointer[i]];
627 interior_nodes[proc]++;
628 }
629 else {
630 // Get the <node,proc> pairs for all border nodes on this processor...
631 // Not efficient at this time...
632 size_t beg = node_to_proc_pointer[i];
633 size_t end = node_to_proc_pointer[i + 1];
634 for (size_t j = beg; j < end; j++) {
635 for (size_t k = beg; k < end; k++) {
636 if (j == k) {
637 continue;
638 }
639 size_t p = node_to_proc[k];
640 border_nodes[p]++;
641 }
642 }
643 }
644 }
645
646 INT global_element_count = global_region.get_property("element_count").get_int();
647
648 // Categorize each element as interior...
649 // Categorize the remaining nodes as border...
650 for (size_t p = 0; p < proc_count; p++) {
651 Ioss::Region *region = proc_region[p];
652
653 INT element_count = region->get_property("element_count").get_int();
654 INT node_count = region->get_property("node_count").get_int();
655 INT border_node_cnt = node_count - interior_nodes[p];
656
657 region->property_add(Ioss::Property("global_node_count", global_node_count));
658 region->property_add(Ioss::Property("global_element_count", global_element_count));
659 region->property_add(Ioss::Property("processor_count", static_cast<int>(proc_count)));
660 region->property_add(Ioss::Property("my_processor", static_cast<int>(p)));
661
662 region->property_add(Ioss::Property("internal_node_count", interior_nodes[p]));
663 region->property_add(Ioss::Property("border_node_count", border_node_cnt));
664 region->property_add(Ioss::Property("internal_element_count", element_count));
665 region->property_add(Ioss::Property("border_element_count", 0));
666
667 // Add commset data... The length of the commset is the number
668 // of <node,proc> pairs for all border nodes.
669 //
670 // For each node on this processor that isn't an interior node,
671 // create the <node,proc> pair...
672 auto *commset =
673 new Ioss::CommSet(region->get_database(), "commset_node", "node", border_nodes[p]);
674 commset->property_add(Ioss::Property("id", 1));
675 region->add(commset);
676 if (debug_level & 2) {
677 fmt::print(stderr, "Commset for processor {} has {} entries.\n", p, border_nodes[p]);
678 }
679 }
680 }
681
682 template <typename INT>
683 void get_nodesets(const Ioss::Region &region, std::vector<Ioss::Region *> &proc_region,
684 const std::vector<int> &node_to_proc,
685 const std::vector<INT> &node_to_proc_pointer)
686 {
687 progress(__func__);
688 // This routine reads the nodesets in the global database;
689 // and defines corresponding nodesets on each processor...
690 size_t proc_count = proc_region.size();
691
692 const auto &ns = region.get_nodesets();
693 size_t set_count = ns.size();
694
695 for (size_t s = 0; s < set_count; s++) {
696 std::vector<INT> pns(proc_count);
697 Ioss::NodeSet *gns = ns[s];
698
699 std::vector<INT> ns_nodes;
700 gns->get_field_data("ids_raw", ns_nodes);
701
702 for (size_t i = 0; i < ns_nodes.size(); i++) {
703 int64_t node = ns_nodes[i] - 1;
704 size_t p_beg = node_to_proc_pointer[node];
705 size_t p_end = node_to_proc_pointer[node + 1];
706 for (size_t j = p_beg; j < p_end; j++) {
707 size_t p = node_to_proc[j];
708 pns[p]++;
709 }
710 }
711
712 const auto &name = ns[s]->name();
713 if (debug_level & 2) {
714 fmt::print(stderr, "\tNodeset {}--", name);
715 }
716 for (size_t p = 0; p < proc_count; p++) {
717 auto *node_set = new Ioss::NodeSet(proc_region[p]->get_database(), name, pns[p]);
718 proc_region[p]->add(node_set);
719 if (debug_level & 2) {
720 fmt::print(stderr, "{}:{}, ", p, pns[p]);
721 }
722 }
723 if (debug_level & 2) {
724 fmt::print(stderr, "\n");
725 }
726 }
727 }
728
729 template <typename INT>
730 void output_nodesets(const Ioss::Region &region, std::vector<Ioss::Region *> &proc_region,
731 const std::vector<int> &node_to_proc,
732 const std::vector<INT> &node_to_proc_pointer, size_t proc_begin,
733 size_t proc_size)
734 {
735 progress(__func__);
736 // This routine reads the nodesets in the global database;
737 // and defines corresponding nodesets on each processor...
738 size_t proc_count = proc_region.size();
739
740 const auto &ns = region.get_nodesets();
741 size_t set_count = ns.size();
742
743 for (size_t s = 0; s < set_count; s++) {
744 if (debug_level & 4) {
745 progress("\tNodeSet " + std::to_string(s + 1));
746 }
747 Ioss::NodeSet *gns = ns[s];
748
749 std::vector<INT> ns_nodes;
750 gns->get_field_data("ids_raw", ns_nodes);
751
752 std::vector<double> ns_df;
753 gns->get_field_data("distribution_factors", ns_df);
754
755 std::vector<std::vector<INT>> pns_nodes(proc_count);
756 std::vector<std::vector<double>> pns_df(proc_count);
757 for (size_t p = proc_begin; p < proc_begin + proc_size; p++) {
758 size_t node_count = proc_region[p]->get_nodesets()[s]->entity_count();
759 pns_nodes[p].reserve(node_count);
760 pns_df[p].reserve(node_count);
761 }
762
763 for (size_t i = 0; i < ns_nodes.size(); i++) {
764 int64_t node = ns_nodes[i] - 1;
765 size_t p_beg = node_to_proc_pointer[node];
766 size_t p_end = node_to_proc_pointer[node + 1];
767 for (size_t j = p_beg; j < p_end; j++) {
768 size_t p = node_to_proc[j];
769 pns_nodes[p].push_back(node + 1);
770 pns_df[p].push_back(ns_df[i]);
771 }
772 }
773
774 for (size_t p = proc_begin; p < proc_begin + proc_size; p++) {
775 Ioss::NodeSet *proc_ns = proc_region[p]->get_nodesets()[s];
776 proc_ns->put_field_data("ids", pns_nodes[p]);
777 proc_ns->put_field_data("distribution_factors", pns_df[p]);
778 proc_progress(p, proc_count);
779 }
780 }
781 }
782
783 template <typename INT>
784 void output_node_map(const Ioss::Region &region, std::vector<Ioss::Region *> &proc_region,
785 const std::vector<int> &node_to_proc,
786 const std::vector<INT> &node_to_proc_pointer, size_t proc_begin,
787 size_t proc_size)
788 {
789 progress(__func__);
790 // This is the processor-local to global-implicit node map...
791 // This maps the 1..#node in the global mesh to each processor...
792 size_t node_count = region.get_property("node_count").get_int();
793 size_t proc_count = proc_region.size();
794
795 std::vector<std::vector<INT>> proc_map(proc_count);
796 for (size_t p = proc_begin; p < proc_begin + proc_size; p++) {
797 size_t pnode_count = proc_region[p]->get_property("node_count").get_int();
798 proc_map[p].reserve(pnode_count);
799 }
800
801 for (size_t i = 0; i < node_count; i++) {
802 size_t p_beg = node_to_proc_pointer[i];
803 size_t p_end = node_to_proc_pointer[i + 1];
804 for (size_t j = p_beg; j < p_end; j++) {
805 size_t p = node_to_proc[j];
806 if (p >= proc_begin && p < proc_begin + proc_size) {
807 proc_map[p].push_back(i + 1);
808 }
809 }
810 }
811
812 for (size_t p = proc_begin; p < proc_begin + proc_size; p++) {
813 Ioss::NodeBlock *nb = proc_region[p]->get_node_blocks()[0];
814 nb->put_field_data("ids", proc_map[p]);
815 proc_map[p].clear();
816 proc_progress(p, proc_count);
817 }
818 }
819
820 template <typename INT>
821 void output_global_node_map(const Ioss::Region &region, std::vector<Ioss::Region *> &proc_region,
822 const std::vector<int> &node_to_proc,
823 const std::vector<INT> &node_to_proc_pointer, size_t proc_begin,
824 size_t proc_size)
825 {
826 progress(__func__);
827 // This is the processor-local to global-implicit node map...
828 // This maps the node_number map (if it exists) in the global mesh
829 // to each processor...
830 std::vector<INT> ids;
831 Ioss::NodeBlock *gnb = region.get_node_blocks()[0];
832 gnb->get_field_data("ids", ids);
833
834 // Check whether the map is sequential (X maps to X);
835 bool sequential = is_sequential(ids);
836 if (!sequential) {
837 fmt::print(stderr, "Node map is not sequential...\n");
838 }
839
840 size_t node_count = region.get_property("node_count").get_int();
841 size_t proc_count = proc_region.size();
842
843 std::vector<std::vector<INT>> proc_map(proc_count);
844 for (size_t p = proc_begin; p < proc_begin + proc_size; p++) {
845 size_t pnode_count = proc_region[p]->get_property("node_count").get_int();
846 proc_map[p].reserve(pnode_count);
847 }
848
849 for (size_t i = 0; i < node_count; i++) {
850 size_t p_beg = node_to_proc_pointer[i];
851 size_t p_end = node_to_proc_pointer[i + 1];
852 for (size_t j = p_beg; j < p_end; j++) {
853 size_t p = node_to_proc[j];
854 if (p >= proc_begin && p < proc_begin + proc_size) {
855 proc_map[p].push_back(ids[i]);
856 }
857 }
858 }
859
860 for (size_t p = proc_begin; p < proc_begin + proc_size; p++) {
861 Ioss::NodeBlock *nb = proc_region[p]->get_node_blocks()[0];
862 nb->put_field_data("ids", proc_map[p]);
863 proc_map[p].clear();
864 proc_progress(p, proc_count);
865 }
866 }
867
868 template <typename INT>
869 void output_element_map(const Ioss::Region &region, std::vector<Ioss::Region *> &proc_region,
870 const std::vector<int> &elem_to_proc, size_t proc_begin, size_t proc_size,
871 INT /* dummy */)
872 {
873 progress(__func__);
874 // map[p][b] = map for block b on processor p
875 size_t proc_count = proc_region.size();
876
877 const auto &ebs = region.get_element_blocks();
878 size_t block_count = ebs.size();
879
880 size_t offset = 0;
881 for (size_t b = 0; b < block_count; b++) {
882 if (debug_level & 4) {
883 progress("\tBlock " + std::to_string(b + 1));
884 }
885 #if 0
886 std::vector<INT> ids;
887 ebs[b]->get_field_data("ids", ids);
888 #endif
889
890 std::vector<std::vector<INT>> map(proc_count);
891 for (size_t p = proc_begin; p < proc_begin + proc_size; p++) {
892 const auto &proc_ebs = proc_region[p]->get_element_blocks();
893 size_t proc_element_count = proc_ebs[b]->entity_count();
894 map[p].reserve(proc_element_count);
895 }
896
897 size_t element_count = ebs[b]->entity_count();
898
899 for (size_t j = 0; j < element_count; j++) {
900 size_t p = elem_to_proc[offset + j];
901 if (p >= proc_begin && p < proc_begin + proc_size) {
902 #if 0
903 map[p].push_back(ids[j]);
904 #else
905 map[p].push_back(offset + j + 1);
906 #endif
907 }
908 }
909 offset += element_count;
910
911 for (size_t p = proc_begin; p < proc_begin + proc_size; p++) {
912 const auto &proc_ebs = proc_region[p]->get_element_blocks();
913 proc_ebs[b]->put_field_data("ids", map[p]);
914 map[p].clear();
915 proc_progress(p, proc_count);
916 }
917 }
918 }
919
920 template <typename INT>
921 void output_coordinates(const Ioss::Region &region, std::vector<Ioss::Region *> &proc_region,
922 const std::vector<int> &node_to_proc,
923 const std::vector<INT> &node_to_proc_pointer, size_t proc_begin,
924 size_t proc_size)
925 {
926 progress(__func__);
927 size_t spatial_dimension = region.get_property("spatial_dimension").get_int();
928
929 std::vector<double> glob_coord_x;
930 std::vector<double> glob_coord_y;
931 std::vector<double> glob_coord_z;
932 Ioss::NodeBlock *gnb = region.get_node_blocks()[0];
933
934 // Distribute nodal coordinates to each processor...
935 // coordinates[p][i] = x,y,z coordinates on processor p
936 size_t processor_count = proc_region.size();
937 std::vector<std::vector<double>> coordinates_x(processor_count);
938 std::vector<std::vector<double>> coordinates_y(processor_count);
939 std::vector<std::vector<double>> coordinates_z(processor_count);
940 for (size_t p = proc_begin; p < proc_begin + proc_size; p++) {
941 size_t pnode_count = proc_region[p]->get_property("node_count").get_int();
942 coordinates_x[p].reserve(pnode_count);
943 coordinates_y[p].reserve(pnode_count);
944 coordinates_z[p].reserve(pnode_count);
945 }
946 progress("\tReserve processor coordinate vectors");
947
948 Ioss::DatabaseIO *db = region.get_database();
949
950 size_t node_count = region.get_property("node_count").get_int();
951
952 if (node_count > partial_count) {
953 int exoid = db->get_file_pointer();
954
955 glob_coord_x.resize(partial_count);
956 glob_coord_y.resize(partial_count);
957 glob_coord_z.resize(partial_count);
958 for (size_t beg = 1; beg <= node_count; beg += partial_count) {
959 size_t count = partial_count;
960 if (beg + count - 1 > node_count) {
961 count = node_count - beg + 1;
962 }
963
964 ex_get_partial_coord(exoid, beg, count, Data(glob_coord_x), Data(glob_coord_y),
965 Data(glob_coord_z));
966 progress("\tpartial_coord: " + std::to_string(beg) + " " + std::to_string(count));
967
968 for (size_t i = 0; i < count; i++) {
969 size_t ii = beg + i - 1;
970 size_t p_beg = node_to_proc_pointer[ii];
971 size_t p_end = node_to_proc_pointer[ii + 1];
972 for (size_t j = p_beg; j < p_end; j++) {
973 size_t p = node_to_proc[j];
974 if (p >= proc_begin && p < proc_begin + proc_size) {
975 coordinates_x[p].push_back(glob_coord_x[i]);
976 coordinates_y[p].push_back(glob_coord_y[i]);
977 coordinates_z[p].push_back(glob_coord_z[i]);
978 }
979 }
980 }
981 }
982 }
983 else {
984 gnb->get_field_data("mesh_model_coordinates_x", glob_coord_x);
985 if (spatial_dimension > 1) {
986 gnb->get_field_data("mesh_model_coordinates_y", glob_coord_y);
987 }
988 if (spatial_dimension > 2) {
989 gnb->get_field_data("mesh_model_coordinates_z", glob_coord_z);
990 }
991 progress("\tRead global mesh_model_coordinates");
992
993 for (size_t i = 0; i < node_count; i++) {
994 size_t p_beg = node_to_proc_pointer[i];
995 size_t p_end = node_to_proc_pointer[i + 1];
996 for (size_t j = p_beg; j < p_end; j++) {
997 size_t p = node_to_proc[j];
998 if (p >= proc_begin && p < proc_begin + proc_size) {
999 coordinates_x[p].push_back(glob_coord_x[i]);
1000 coordinates_y[p].push_back(glob_coord_y[i]);
1001 coordinates_z[p].push_back(glob_coord_z[i]);
1002 }
1003 }
1004 }
1005 }
1006 progress("\tPopulate processor coordinate vectors");
1007 Ioss::Utils::clear(glob_coord_x);
1008 Ioss::Utils::clear(glob_coord_y);
1009 Ioss::Utils::clear(glob_coord_z);
1010
1011 for (size_t p = proc_begin; p < proc_begin + proc_size; p++) {
1012 Ioss::NodeBlock *nb = proc_region[p]->get_node_blocks()[0];
1013 nb->put_field_data("mesh_model_coordinates_x", coordinates_x[p]);
1014 if (spatial_dimension > 1) {
1015 nb->put_field_data("mesh_model_coordinates_y", coordinates_y[p]);
1016 }
1017 if (spatial_dimension > 2) {
1018 nb->put_field_data("mesh_model_coordinates_z", coordinates_z[p]);
1019 }
1020 proc_progress(p, processor_count);
1021 }
1022 progress("\tOutput processor coordinate vectors");
1023 }
1024
1025 // Output a component at a time...
1026 template <typename INT>
1027 void output_coordinates_c(const Ioss::Region &region, std::vector<Ioss::Region *> &proc_region,
1028 const std::vector<int> &node_to_proc,
1029 const std::vector<INT> &node_to_proc_pointer, size_t proc_begin,
1030 size_t proc_size)
1031 {
1032 progress(__func__);
1033 std::vector<double> glob_coord;
1034 Ioss::NodeBlock *gnb = region.get_node_blocks()[0];
1035
1036 std::array<std::string, 3> field_name{"mesh_model_coordinates_x", "mesh_model_coordinates_y",
1037 "mesh_model_coordinates_z"};
1038 // Distribute nodal coordinates to each processor...
1039 // coordinates[p][i] = x,y,z coordinates on processor p
1040 size_t processor_count = proc_region.size();
1041 std::vector<std::vector<double>> coordinates(processor_count);
1042
1043 Ioss::DatabaseIO *db = region.get_database();
1044
1045 size_t node_count = region.get_property("node_count").get_int();
1046
1047 for (size_t p = proc_begin; p < proc_begin + proc_size; p++) {
1048 size_t pnode_count = proc_region[p]->get_property("node_count").get_int();
1049 coordinates[p].reserve(pnode_count);
1050 }
1051 progress("\tReserve processor coordinate vectors");
1052
1053 size_t spatial_dimension = region.get_property("spatial_dimension").get_int();
1054
1055 for (size_t comp = 0; comp < spatial_dimension; comp++) {
1056 for (size_t p = proc_begin; p < proc_begin + proc_size; p++) {
1057 coordinates[p].resize(0);
1058 }
1059
1060 if (node_count > partial_count) {
1061 int exoid = db->get_file_pointer();
1062
1063 glob_coord.resize(partial_count);
1064 for (size_t beg = 1; beg <= node_count; beg += partial_count) {
1065 size_t count = partial_count;
1066 if (beg + count - 1 > node_count) {
1067 count = node_count - beg + 1;
1068 }
1069
1070 switch (comp) {
1071 case 0:
1072 ex_get_partial_coord(exoid, beg, count, Data(glob_coord), nullptr, nullptr);
1073 break;
1074 case 1:
1075 ex_get_partial_coord(exoid, beg, count, nullptr, Data(glob_coord), nullptr);
1076 break;
1077 case 2:
1078 ex_get_partial_coord(exoid, beg, count, nullptr, nullptr, Data(glob_coord));
1079 break;
1080 }
1081 progress("\tpartial_coord: " + std::to_string(beg) + " " + std::to_string(count));
1082
1083 for (size_t i = 0; i < count; i++) {
1084 size_t ii = beg + i - 1;
1085 size_t p_beg = node_to_proc_pointer[ii];
1086 size_t p_end = node_to_proc_pointer[ii + 1];
1087 for (size_t j = p_beg; j < p_end; j++) {
1088 size_t p = node_to_proc[j];
1089 if (p >= proc_begin && p < proc_begin + proc_size) {
1090 coordinates[p].push_back(glob_coord[i]);
1091 }
1092 }
1093 }
1094 }
1095 }
1096 else {
1097 gnb->get_field_data(field_name[comp], glob_coord);
1098 progress("\tRead global mesh_model_coordinates");
1099
1100 for (size_t i = 0; i < node_count; i++) {
1101 size_t p_beg = node_to_proc_pointer[i];
1102 size_t p_end = node_to_proc_pointer[i + 1];
1103 for (size_t j = p_beg; j < p_end; j++) {
1104 size_t p = node_to_proc[j];
1105 if (p >= proc_begin && p < proc_begin + proc_size) {
1106 coordinates[p].push_back(glob_coord[i]);
1107 }
1108 }
1109 }
1110 }
1111 progress("\tPopulate processor coordinate vectors");
1112 Ioss::Utils::clear(glob_coord);
1113
1114 for (size_t p = proc_begin; p < proc_begin + proc_size; p++) {
1115 Ioss::NodeBlock *nb = proc_region[p]->get_node_blocks()[0];
1116 nb->put_field_data(field_name[comp], coordinates[p]);
1117 proc_progress(p, processor_count);
1118 }
1119 }
1120 progress("\tOutput processor coordinate vectors");
1121 }
1122
1123 template <typename INT>
1124 void output_connectivity(const Ioss::Region &region, std::vector<Ioss::Region *> &proc_region,
1125 const std::vector<int> &elem_to_proc, size_t proc_begin,
1126 size_t proc_size, INT /*dummy*/)
1127 {
1128 // Read connectivity and partition to each processor/block.
1129 // connectvity[p][b] = connectivity for block b on processor p
1130
1131 progress(__func__);
1132 const auto &ebs = region.get_element_blocks();
1133 size_t block_count = ebs.size();
1134
1135 size_t processor_count = proc_region.size();
1136
1137 Ioss::DatabaseIO *db = region.get_database();
1138
1139 std::vector<INT> glob_conn;
1140 size_t offset = 0;
1141
1142 for (size_t b = 0; b < block_count; b++) {
1143 std::vector<std::vector<INT>> connectivity(processor_count);
1144 size_t element_count = ebs[b]->entity_count();
1145 size_t element_nodes = ebs[b]->topology()->number_nodes();
1146 size_t block_id = ebs[b]->get_property("id").get_int();
1147
1148 for (size_t p = proc_begin; p < proc_begin + proc_size; p++) {
1149 const auto &pebs = proc_region[p]->get_element_blocks();
1150 size_t pelement_count = pebs[b]->entity_count();
1151 size_t pelement_nodes = pebs[b]->topology()->number_nodes();
1152 connectivity[p].reserve(pelement_count * pelement_nodes); // Use reserve, not resize
1153 }
1154
1155 // Do a 'partial_count' elements at a time...
1156 if (element_count >= partial_count) {
1157 int exoid = db->get_file_pointer();
1158
1159 glob_conn.resize(partial_count * element_nodes);
1160 for (size_t beg = 1; beg <= element_count; beg += partial_count) {
1161 size_t count = partial_count;
1162 if (beg + count - 1 > element_count) {
1163 count = element_count - beg + 1;
1164 }
1165
1166 ex_get_partial_conn(exoid, EX_ELEM_BLOCK, block_id, beg, count, Data(glob_conn), nullptr,
1167 nullptr);
1168 progress(fmt::format("\tpartial_conn-- start: {}\tcount: {}", fmt::group_digits(beg),
1169 fmt::group_digits(count)));
1170
1171 size_t el = 0;
1172 for (size_t j = 0; j < count; j++) {
1173 size_t p = elem_to_proc[offset + j];
1174 if (p >= proc_begin && p < proc_begin + proc_size) {
1175 for (size_t k = 0; k < element_nodes; k++) {
1176 connectivity[p].push_back(glob_conn[el++]);
1177 }
1178 }
1179 else {
1180 el += element_nodes;
1181 }
1182 }
1183 offset += count;
1184 }
1185 }
1186 else {
1187 ebs[b]->get_field_data("connectivity_raw", glob_conn);
1188
1189 size_t el = 0;
1190 for (size_t j = 0; j < element_count; j++) {
1191 size_t p = elem_to_proc[offset + j];
1192 if (p >= proc_begin && p < proc_begin + proc_size) {
1193 for (size_t k = 0; k < element_nodes; k++) {
1194 connectivity[p].push_back(glob_conn[el++]);
1195 }
1196 }
1197 else {
1198 el += element_nodes;
1199 }
1200 }
1201 offset += element_count;
1202 }
1203
1204 for (size_t p = proc_begin; p < proc_begin + proc_size; p++) {
1205 const auto &proc_ebs = proc_region[p]->get_element_blocks();
1206 Ioss::ElementBlock *eb = proc_ebs[b];
1207 eb->put_field_data("connectivity", connectivity[p]);
1208 }
1209 }
1210 }
1211
1212 template <typename INT>
1213 void get_proc_elem_block_count(const Ioss::Region &region, std::vector<int> &elem_to_proc,
1214 std::vector<std::vector<INT>> &proc_elem_block_cnt)
1215 {
1216 progress(__func__);
1217 const auto &ebs = region.get_element_blocks();
1218 size_t block_count = ebs.size();
1219 size_t begin = 0;
1220 for (size_t i = 0; i < block_count; i++) {
1221 size_t end = begin + ebs[i]->entity_count();
1222 for (size_t j = begin; j < end; j++) {
1223 size_t processor = elem_to_proc[j];
1224 proc_elem_block_cnt[i][processor]++;
1225 }
1226 begin = end;
1227 }
1228
1229 size_t processor_count = proc_elem_block_cnt[0].size();
1230 for (size_t i = 0; i < processor_count; i++) {
1231 size_t sum = 0;
1232 for (size_t j = 0; j < block_count; j++) {
1233 sum += proc_elem_block_cnt[j][i];
1234 }
1235 proc_elem_block_cnt[block_count][i] = sum;
1236 if (debug_level & 2) {
1237 fmt::print(stderr, "\tProcessor {} has {} elements.\n", fmt::group_digits(i),
1238 fmt::group_digits(sum));
1239 }
1240 }
1241 }
1242
1243 template <typename INT>
1244 void get_node_to_proc(Ioss::Region &region, std::vector<Ioss::Region *> &proc_region,
1245 const std::vector<int> &elem_to_proc, std::vector<int> &node_to_proc,
1246 std::vector<INT> &node_to_proc_pointer)
1247 {
1248 progress(__func__);
1249 // Process each element block connectivity to get the node_to_proc mapping.
1250
1251 // The 'node_to_proc_pointer' vector maps the processor span in the
1252 // node_to_proc vector. The processors that node 'node' (0-based)
1253 // is on are:
1254 // * begin = node_to_proc_pointer[node]
1255 // * end = node_to_proc_pointer[node+1]
1256 // * proc_list = node_to_proc[begin] .. node_to_proc[end-1]
1257 //
1258
1259 size_t proc_count = proc_region.size();
1260
1261 size_t node_count = region.get_property("node_count").get_int();
1262 if (node_count == 0) {
1263 // Should never happen, but makes static analyzers happy...
1264 return;
1265 }
1266
1267 std::vector<std::vector<int>> proc_node(node_count);
1268
1269 // Assume that the majority of nodes will be on 2 or less
1270 // processors (hopefully, most are on 1).
1271 // Preallocate the proc_node[node] vector to 2 to minimize
1272 // resizes... Use 'reserve' instead of 'resize'
1273 for (size_t i = 0; i < node_count; i++) {
1274 proc_node[i].reserve(2);
1275 }
1276 progress("\tProc_node reserved");
1277
1278 IOSS_MAYBE_UNUSED size_t sum_on_proc_count = 0;
1279 Ioss::DatabaseIO *db = region.get_database();
1280
1281 const auto &ebs = region.get_element_blocks();
1282 size_t block_count = ebs.size();
1283 size_t offset = 0;
1284
1285 std::vector<size_t> on_proc_count(proc_count);
1286 for (size_t b = 0; b < block_count; b++) {
1287 std::vector<INT> glob_conn;
1288 size_t element_count = ebs[b]->entity_count();
1289 size_t element_nodes = ebs[b]->topology()->number_nodes();
1290 size_t block_id = ebs[b]->get_property("id").get_int();
1291
1292 // Do a 'partial_count' elements at a time...
1293 if (element_count >= partial_count) {
1294 int exoid = db->get_file_pointer();
1295
1296 glob_conn.resize(partial_count * element_nodes);
1297 for (size_t beg = 1; beg <= element_count; beg += partial_count) {
1298 size_t count = partial_count;
1299 if (beg + count - 1 > element_count) {
1300 count = element_count - beg + 1;
1301 }
1302
1303 ex_get_partial_conn(exoid, EX_ELEM_BLOCK, block_id, beg, count, Data(glob_conn), nullptr,
1304 nullptr);
1305 progress(fmt::format("\tpartial_conn-- start: {}\tcount: {}", fmt::group_digits(beg),
1306 fmt::group_digits(count)));
1307 populate_proc_node(count, offset, element_nodes, elem_to_proc, glob_conn, proc_node,
1308 on_proc_count);
1309 offset += count;
1310 }
1311 }
1312 else {
1313 ebs[b]->get_field_data("connectivity_raw", glob_conn);
1314 populate_proc_node(element_count, offset, element_nodes, elem_to_proc, glob_conn, proc_node,
1315 on_proc_count);
1316 offset += element_count;
1317 }
1318 }
1319 size_t spatial_dimension = region.get_property("spatial_dimension").get_int();
1320 for (size_t p = 0; p < proc_count; p++) {
1321 auto *nb = new Ioss::NodeBlock(proc_region[p]->get_database(), "node_block1",
1322 on_proc_count[p], spatial_dimension);
1323 proc_region[p]->add(nb);
1324 if (debug_level & 2) {
1325 fmt::print(stderr, "\tProcessor {} has {} nodes.\n", fmt::group_digits(p),
1326 fmt::group_digits(on_proc_count[p]));
1327 }
1328 sum_on_proc_count += on_proc_count[p];
1329 }
1330 progress("\tProc_node populated");
1331
1332 // Have data for each node showing which processors it is on...
1333 // proc_node[node].size() is number of processors for this node...
1334 node_to_proc_pointer.reserve(node_count + 1);
1335
1336 std::vector<size_t> proc_histo(17);
1337
1338 size_t node_to_proc_pointer_size = 0;
1339 for (size_t i = 0; i < node_count; i++) {
1340 size_t num_procs = proc_node[i].size();
1341 if (num_procs == 0) {
1342 fmt::print(stderr, "WARNING: Node {} is not connected to any elements.\n",
1343 fmt::group_digits(i + 1));
1344 }
1345 else if (num_procs < proc_histo.size()) {
1346 proc_histo[num_procs]++;
1347 }
1348 else {
1349 proc_histo[0]++;
1350 }
1351
1352 node_to_proc_pointer.push_back(node_to_proc_pointer_size);
1353 node_to_proc_pointer_size += num_procs;
1354 }
1355 // Output histogram..
1356 fmt::print(stderr, "Processor count per node histogram:\n");
1357 for (size_t i = 1; i < proc_histo.size(); i++) {
1358 if (proc_histo[i] > 0) {
1359 fmt::print(stderr, "\tNodes on {:2} processors = {:12}\t({:2})%\n", fmt::group_digits(i),
1360 fmt::group_digits(proc_histo[i]),
1361 (proc_histo[i] * 100 + node_count / 2) / node_count);
1362 }
1363 }
1364 if (proc_histo[0] > 0) {
1365 fmt::print(stderr, "\tNodes on {} or more processors = {}\t({:2})%\n",
1366 fmt::group_digits(proc_histo.size()), fmt::group_digits(proc_histo[0]),
1367 (proc_histo[0] * 100 + node_count / 2) / node_count);
1368 }
1369 fmt::print(stderr, "\n");
1370
1371 node_to_proc_pointer.push_back(node_to_proc_pointer_size);
1372 node_to_proc.reserve(node_to_proc_pointer_size);
1373 progress("\tNode_to_proc reserved");
1374 assert(sum_on_proc_count == node_to_proc_pointer_size);
1375
1376 for (auto &pn : proc_node) {
1377 size_t num_procs = pn.size();
1378 for (size_t p = 0; p < num_procs; p++) {
1379 node_to_proc.push_back(pn[p]);
1380 }
1381 }
1382 assert(node_to_proc.size() == node_to_proc_pointer_size);
1383 progress("\tNode_to_proc populated");
1384 }
1385
1386 template <typename INT>
1387 void slice(Ioss::Region &region, const std::string &nemfile, SystemInterface &interFace,
1388 INT dummy)
1389 {
1390 progress(__func__);
1391 bool create_split_files = !interFace.outputDecompMap_ && !interFace.outputDecompField_;
1392
1393 std::vector<Ioss::Region *> proc_region;
1394 if (create_split_files) {
1395 proc_region.resize(interFace.processor_count());
1396 }
1397 bool ints64 = (sizeof(INT) == 8);
1398
1399 Ioss::PropertyManager properties = set_properties(interFace);
1400
1401 Ioss::chain_t<INT> element_chains;
1402 std::vector<float> weights;
1403 if (interFace.lineDecomp_) {
1404 element_chains =
1405 Ioss::generate_element_chains(region, interFace.lineSurfaceList_, debug_level, dummy);
1406 progress("Ioss::generate_element_chains");
1407
1408 if (interFace.decomposition_method() == "rcb" || interFace.decomposition_method() == "rib" ||
1409 interFace.decomposition_method() == "hsfc") {
1410 weights = Ioss::DecompUtils::line_decomp_weights(
1411 element_chains, region.get_property("element_count").get_int());
1412 progress("generate_element_weights");
1413 }
1414 }
1415
1416 if (weights.empty()) {
1417 weights.resize(region.get_property("element_count").get_int(), 1);
1418 }
1419
1420 double start = seacas_timer();
1421 auto elem_to_proc = decompose_elements(region, interFace, weights, dummy);
1422 double end = seacas_timer();
1423 fmt::print(stderr, "Decompose elements = {:.5}\n", end - start);
1424 progress("exit decompose_elements");
1425
1426 if (interFace.lineDecomp_) {
1427 // Make sure all elements on a chain are on the same processor rank...
1428 Ioss::DecompUtils::line_decomp_modify(element_chains, elem_to_proc,
1429 interFace.processor_count());
1430 }
1431
1432 if (debug_level & 32) {
1433 auto work_per_rank =
1434 Ioss::DecompUtils::get_work_per_rank(elem_to_proc, interFace.processor_count());
1435 auto avg_median = Ioss::DecompUtils::output_decomposition_statistics(work_per_rank);
1436 Ioss::DecompUtils::output_histogram(work_per_rank, avg_median.first, avg_median.second);
1437 }
1438
1439 if (!create_split_files) {
1440 Ioss::DatabaseIO *dbo = Ioss::IOFactory::create(
1441 "exodus", nemfile, Ioss::WRITE_RESTART, Ioss::ParallelUtils::comm_world(), properties);
1442 if (dbo == nullptr || !dbo->ok(true)) {
1443 std::exit(EXIT_FAILURE);
1444 }
1445 if (ints64) {
1446 dbo->set_int_byte_size_api(Ioss::USE_INT64_API);
1447 }
1448
1449 // NOTE: 'output_region' owns 'dbo' pointer at this time
1450 Ioss::Region output_region(dbo, "region_2");
1451
1452 // Set the qa information...
1453 output_region.property_add(Ioss::Property(std::string("code_name"), qainfo[0]));
1454 output_region.property_add(Ioss::Property(std::string("code_version"), qainfo[2]));
1455
1456 Ioss::MeshCopyOptions options{};
1457 options.ints_64_bit = sizeof(INT) == 64;
1458 options.delete_timesteps = true;
1459 options.data_storage_type = 1;
1460 options.verbose = true;
1461
1462 // Copy mesh portion of input region to the output region
1463 Ioss::copy_database(region, output_region, options);
1464
1465 // KLUGE: The metadata has already been written on
1466 // output_region, but we couldn't define the maps until now, so
1467 // need to update the metadata with map information and hope
1468 // that no other maps exist on the database...
1469 if (interFace.outputDecompMap_) {
1470 bool line_decomp = interFace.lineDecomp_;
1471 add_decomp_map(output_region, interFace.decomposition_variable(), line_decomp);
1472 output_decomp_map(output_region, elem_to_proc, element_chains,
1473 interFace.decomposition_variable(), line_decomp);
1474 progress("output_decomp_map");
1475 }
1476
1477 if (interFace.outputDecompField_) {
1478 bool line_decomp = interFace.lineDecomp_;
1479 add_decomp_field(output_region, interFace.decomposition_variable(), line_decomp);
1480 output_decomp_field(output_region, elem_to_proc, element_chains,
1481 interFace.decomposition_variable(), line_decomp);
1482 progress("output_decomp_field");
1483 }
1484
1485 return;
1486 }
1487
1488 bool close_files = interFace.processor_count() + 1 > interFace.max_files();
1489 for (size_t i = 0; i < interFace.processor_count(); i++) {
1490 std::string outfile = Ioss::Utils::decode_filename(nemfile, i, interFace.processor_count());
1491 Ioss::DatabaseIO *dbo = Ioss::IOFactory::create(
1492 "exodus", outfile, Ioss::WRITE_RESTART, Ioss::ParallelUtils::comm_world(), properties);
1493 if (ints64) {
1494 dbo->set_int_byte_size_api(Ioss::USE_INT64_API);
1495 }
1496
1497 proc_region[i] = new Ioss::Region(dbo);
1498 proc_region[i]->begin_mode(Ioss::STATE_DEFINE_MODEL);
1499 if (close_files) {
1500 proc_region[i]->get_database()->closeDatabase();
1501 }
1502 }
1503
1504 start = seacas_timer();
1505 // Build the proc_elem_block_cnt[i][j] vector.
1506 // Gives number of elements in block i on processor j
1507 size_t block_count = region.get_property("element_block_count").get_int();
1508 std::vector<std::vector<INT>> proc_elem_block_cnt(block_count + 1);
1509 for (auto &pebc : proc_elem_block_cnt) {
1510 pebc.resize(interFace.processor_count());
1511 }
1512 get_proc_elem_block_count(region, elem_to_proc, proc_elem_block_cnt);
1513 end = seacas_timer();
1514
1515 fmt::print(stderr, "Calculate elements per element block on each processor = {:.5}\n",
1516 end - start);
1517
1518 // Create element blocks for each processor...
1519 for (size_t p = 0; p < interFace.processor_count(); p++) {
1520 const auto &ebs = region.get_element_blocks();
1521 size_t bc = ebs.size();
1522 for (size_t b = 0; b < bc; b++) {
1523 std::string type = ebs[b]->topology()->name();
1524 auto *eb = new Ioss::ElementBlock(proc_region[p]->get_database(), ebs[b]->name(), type,
1525 proc_elem_block_cnt[b][p]);
1526 proc_region[p]->add(eb);
1527 }
1528 }
1529
1530 // Now that we have the elements on each processor and the element
1531 // blocks those elements are in, can generate the node to proc list...
1532 start = seacas_timer();
1533 std::vector<int> node_to_proc;
1534 std::vector<INT> node_to_proc_pointer;
1535 get_node_to_proc(region, proc_region, elem_to_proc, node_to_proc, node_to_proc_pointer);
1536 end = seacas_timer();
1537 fmt::print(stderr, "Node Categorization Time = {:.5}\n", end - start);
1538
1539 // Communication map data -- interior/border nodes
1540 start = seacas_timer();
1541 define_communication_data(region, proc_region, node_to_proc, node_to_proc_pointer);
1542 end = seacas_timer();
1543 fmt::print(stderr, "Communication Data Definitions = {:.5}\n", end - start);
1544
1545 // Determine nodeset distribution to processor regions.
1546 start = seacas_timer();
1547 get_nodesets(region, proc_region, node_to_proc, node_to_proc_pointer);
1548 end = seacas_timer();
1549 fmt::print(stderr, "Get nodeset data = {:.5}\n", end - start);
1550
1551 start = seacas_timer();
1552 get_sidesets(region, proc_region, elem_to_proc, (INT)0);
1553 end = seacas_timer();
1554 fmt::print(stderr, "Get sideset data = {:.5}\n", end - start);
1555
1556 start = seacas_timer();
1557 double start_comb = start;
1558 fmt::print(stderr, "Begin writing output files\n");
1559 size_t proc_count = interFace.processor_count();
1560
1561 // Output in processor chunks of size <= max_files so can keep all files open....
1562 size_t max_files = interFace.max_files();
1563 size_t chunks = (proc_count + max_files - 1) / max_files;
1564 size_t size_per_chunk = (proc_count + chunks - 1) / chunks;
1565 if (chunks > 1) {
1566 fmt::print(stderr,
1567 "\nMax open files = {}; processing files in {} chunks of size {} to maximize "
1568 "performance.\n",
1569 max_files, chunks, size_per_chunk);
1570 }
1571 for (size_t chunk = 0; chunk < chunks; chunk++) {
1572 size_t proc_begin = chunk * size_per_chunk;
1573 size_t proc_size = size_per_chunk;
1574 if (proc_begin + proc_size > proc_count) {
1575 proc_size = proc_count - proc_begin;
1576 }
1577 fmt::print(stderr, "\nProcessor range {} to {}\n", fmt::group_digits(proc_begin),
1578 fmt::group_digits(proc_begin + proc_size - 1));
1579
1580 for (size_t p = proc_begin; p < proc_begin + proc_size; p++) {
1581 Ioss::transfer_coordinate_frames(region, *proc_region[p]);
1582 Ioss::transfer_assemblies(region, *proc_region[p], Ioss::MeshCopyOptions{}, 0);
1583 proc_region[p]->synchronize_id_and_name(&region);
1584 proc_region[p]->end_mode(Ioss::STATE_DEFINE_MODEL);
1585 if (interFace.lineDecomp_) {
1586 add_chain_maps(*proc_region[p]);
1587 }
1588 proc_region[p]->begin_mode(Ioss::STATE_MODEL);
1589 proc_progress(p, proc_count);
1590 }
1591 end = seacas_timer();
1592 fmt::print(stderr, "\tDefine output databases = {:.5}\n", end - start);
1593
1594 // Generate and output node map...
1595 #if 1
1596 start = seacas_timer();
1597 output_node_map(region, proc_region, node_to_proc, node_to_proc_pointer, proc_begin,
1598 proc_size);
1599 end = seacas_timer();
1600 fmt::print(stderr, "\tNode Map Output = {:.5}\n", end - start);
1601 #else
1602 start = seacas_timer();
1603 output_global_node_map(region, proc_region, node_to_proc, node_to_proc_pointer, proc_begin,
1604 proc_size);
1605 end = seacas_timer();
1606 fmt::print(stderr, "\tGlobal Node Map Output = {:.5}\n", end - start);
1607 #endif
1608
1609 start = seacas_timer();
1610 output_element_map(region, proc_region, elem_to_proc, proc_begin, proc_size, (INT)1);
1611 end = seacas_timer();
1612 fmt::print(stderr, "\tElement Map Output = {:.5}\n", end - start);
1613
1614 start = seacas_timer();
1615 output_communication_map(region, proc_region, node_to_proc, node_to_proc_pointer, proc_begin,
1616 proc_size);
1617 end = seacas_timer();
1618 fmt::print(stderr, "\tCommunication map Output = {:.5}\n", end - start);
1619
1620 start = seacas_timer();
1621 output_connectivity(region, proc_region, elem_to_proc, proc_begin, proc_size, (INT)1);
1622 end = seacas_timer();
1623 fmt::print(stderr, "\tConnectivity Output = {:.5}\n", end - start);
1624
1625 start = seacas_timer();
1626 #if 0
1627 output_coordinates(region, proc_region, node_to_proc, node_to_proc_pointer, proc_begin,
1628 proc_size);
1629 #else
1630 output_coordinates_c(region, proc_region, node_to_proc, node_to_proc_pointer, proc_begin,
1631 proc_size);
1632 #endif
1633 end = seacas_timer();
1634 fmt::print(stderr, "\tCoordinates Output = {:.5}\n", end - start);
1635
1636 start = seacas_timer();
1637 output_nodesets(region, proc_region, node_to_proc, node_to_proc_pointer, proc_begin,
1638 proc_size);
1639 end = seacas_timer();
1640 fmt::print(stderr, "\tNodeset Output = {:.5}\n", end - start);
1641
1642 start = seacas_timer();
1643 output_sidesets(region, proc_region, elem_to_proc, proc_begin, proc_size, (INT)0);
1644 end = seacas_timer();
1645 fmt::print(stderr, "\tSideset Output = {:.5}\n", end - start);
1646
1647 if (interFace.lineDecomp_) {
1648 start = seacas_timer();
1649 output_chain_maps(proc_region, element_chains, elem_to_proc, proc_begin, proc_size, (INT)0);
1650 end = seacas_timer();
1651 fmt::print(stderr, "\tChain Map Output = {:.5}\n", end - start);
1652 }
1653
1654 // Close all files...
1655 start = seacas_timer();
1656 for (size_t p = proc_begin; p < proc_begin + proc_size; p++) {
1657 proc_region[p]->end_mode(Ioss::STATE_MODEL);
1658 delete proc_region[p];
1659 }
1660 end = seacas_timer();
1661 fmt::print(stderr, "\tClose and finalize processor {} to {} output databases = {:.5}\n",
1662 proc_begin, proc_begin + proc_size - 1, end - start);
1663 }
1664 end = seacas_timer();
1665 fmt::print(stderr, "\nTotal time to write output files = {:.5} ({:.5} per file)\n",
1666 end - start_comb, (end - start_comb) / interFace.processor_count());
1667 }
1668
1669 void filename_substitution(std::string &filename, const SystemInterface &interFace)
1670 {
1671 // See if filename contains "%P" which is replaced by the number of processors...
1672 // Assumes that %P only occurs once...
1673 // filename is changed.
1674 size_t pos = filename.find("%P");
1675 if (pos != std::string::npos) {
1676 // Found the characters... Replace with the processor count...
1677 size_t num_proc = interFace.processor_count();
1678 std::string tmp(filename, 0, pos);
1679 tmp += std::to_string(num_proc);
1680 tmp += filename.substr(pos + 2);
1681 filename = tmp;
1682 }
1683
1684 // If contains %M, replace with the decomposition method.
1685 pos = filename.find("%M");
1686 if (pos != std::string::npos) {
1687 // Found the characters... Replace with the input file basename...
1688 const std::string &method_name = interFace.decomposition_method();
1689 std::string tmp(filename, 0, pos);
1690 tmp += method_name;
1691 tmp += filename.substr(pos + 2);
1692 filename = tmp;
1693 }
1694 }
1695
1696 } // namespace