Debian Codegraph @

packages/seacas/applications/epu/epu.C

raw
1 /*
2 * Copyright(C) 1999-2025 National Technology & Engineering Solutions
3 * of Sandia, LLC (NTESS). Under the terms of Contract DE-NA0003525 with
4 * NTESS, the U.S. Government retains certain rights in this software.
5 *
6 * See packages/seacas/LICENSE for details
7 */
8 // concatenates EXODUS/GENESIS output from parallel processors to a single file
9
10 #include <algorithm>
11 #include <array>
12 #include <cfloat>
13 #include <climits>
14 #include <cmath>
15 #include <copy_string_cpp.h>
16 #include <cstddef>
17 #include <cstdint>
18 #include <cstdlib>
19 #include <ctime>
20 #include <exception>
21 #include <fmt/chrono.h>
22 #include <fmt/ostream.h>
23 #include <limits>
24 #include <numeric>
25 #include <set>
26 #include <stdexcept>
27 #include <string>
28 #include <utility>
29 #include <vector>
30
31 #include "format_time.h"
32 #include "open_file_limit.h"
33 #include "sys_info.h"
34 #include "time_stamp.h"
35 #include "vector_data.h"
36
37 #define USE_STD_SORT 1
38 #if !USE_STD_SORT
39 #include "pdqsort.h"
40 #endif
41 #include "hwm.h"
42
43 // Enable SMART_ASSERT even in Release mode...
44 #define SMART_ASSERT_DEBUG_MODE 1
45 #include "smart_assert.h"
46
47 #include <exodusII.h>
48
49 using StringVector = std::vector<std::string>;
50
51 #include "EP_ExodusEntity.h"
52 #include "EP_ExodusFile.h"
53 #include "EP_Internals.h"
54 #include "EP_ObjectType.h"
55 #include "EP_SystemInterface.h"
56 #include "EP_Variables.h"
57 #include "EP_Version.h"
58
59 #if EX_API_VERS_NODOT <= 467
60 #error "Requires exodusII version 4.68 or later"
61 #endif
62
63 #include "add_to_log.h"
64
65 // The main program templated to permit float/double transfer.
66 template <typename T, typename INT>
67 int epu(Excn::SystemInterface &interFace, int start_part, int part_count, int cycle);
68
69 class mpi
70 {
71 public:
72 mpi(int argc, char *argv[])
73 {
74 #if ENABLE_PARALLEL_EPU
75 MPI_Init(&argc, &argv);
76 MPI_Comm_rank(MPI_COMM_WORLD, &rank); // CHECK: ALLOW MPI_COMM_WORLD
77 MPI_Comm_size(MPI_COMM_WORLD, &epu_proc_count); // CHECK: ALLOW MPI_COMM_WORLD
78 #else
79 (void)(argc);
80 (void)(argv);
81 #endif
82 }
83
84 ~mpi()
85 {
86 #if ENABLE_PARALLEL_EPU
87 MPI_Finalize();
88 #endif
89 }
90
91 int rank{0};
92 int epu_proc_count{1};
93 };
94
95 using ExodusIdVector = std::vector<ex_entity_id>;
96 template <typename T> using MasterValueVector = std::vector<T>;
97
98 extern double seacas_timer();
99 namespace {
100 unsigned int debug_level = 0;
101 const double FILL_VALUE = FLT_MAX;
102 int rank = 0;
103 std::string tsFormat = "[{:%H:%M:%S}] ";
104
105 int get_width(int max_value);
106
107 void LOG(std::string_view message)
108 {
109 if ((debug_level & 1) != 0u) {
110 fmt::print("{}", time_stamp(tsFormat));
111 }
112 if (rank == 0) {
113 fmt::print("{}", message);
114 }
115 }
116
117 [[noreturn]] void exodus_error(int lineno)
118 {
119 auto errmsg = fmt::format(
120 "Exodus error ({}) {} at line {} in file epu.C. Please report to [email protected] "
121 "if you need help.",
122 exerrval, ex_strerror(exerrval), lineno);
123
124 ex_err(nullptr, nullptr, EX_PRTLASTMSG);
125 throw std::runtime_error(errmsg);
126 }
127
128 template <typename T> void clear(std::vector<T> &vec)
129 {
130 vec.clear();
131 vec.shrink_to_fit();
132 SMART_ASSERT(vec.capacity() == 0);
133 }
134
135 ex_entity_type exodus_object_type(const Excn::ObjectType &epu_type)
136 {
137 switch (epu_type) {
138 case Excn::ObjectType::EBLK: return EX_ELEM_BLOCK;
139 case Excn::ObjectType::SSET: return EX_SIDE_SET;
140 case Excn::ObjectType::NSET: return EX_NODE_SET;
141 case Excn::ObjectType::EDBLK: return EX_EDGE_BLOCK;
142 case Excn::ObjectType::FABLK: return EX_FACE_BLOCK;
143 default:
144 throw std::runtime_error("Invalid Object Type in exodus_object_type: " +
145 std::to_string(static_cast<int>(epu_type)));
146 }
147 }
148
149 char **get_name_array(int size, int length)
150 {
151 char **names = nullptr;
152 if (size > 0) {
153 names = new char *[size];
154 for (int i = 0; i < size; i++) {
155 names[i] = new char[length + 1];
156 std::memset(names[i], '\0', length + 1);
157 }
158 }
159 return names;
160 }
161
162 void free_name_array(char **names, int size)
163 {
164 for (int i = 0; i < size; i++) {
165 delete[] names[i];
166 }
167 delete[] names;
168 names = nullptr;
169 }
170
171 bool case_compare(const std::string &s1, const std::string &s2);
172
173 template <typename INT>
174 void get_id_map(int exoid, ex_entity_type type, ex_inquiry inq_type, std::vector<INT> &ids)
175 {
176 // Check whether there is a "original_global_id_map" map on
177 // the database. If so, use it instead of the "elem_num_map".
178 bool map_read = false;
179 int map_count = ex_inquire_int(exoid, inq_type);
180 if (map_count > 0) {
181 char **names = get_name_array(map_count, Excn::ExodusFile::max_name_length());
182 int error = ex_get_names(exoid, type, names);
183 if (error < 0) {
184 exodus_error(__LINE__);
185 }
186
187 for (int i = 0; i < map_count; i++) {
188 if (case_compare(names[i], "original_global_id_map")) {
189 error = ex_get_num_map(exoid, type, i + 1, Data(ids));
190 if (error < 0) {
191 exodus_error(__LINE__);
192 }
193 map_read = true;
194 break;
195 }
196 }
197 free_name_array(names, map_count);
198 }
199
200 if (!map_read) {
201 int error = ex_get_id_map(exoid, type, Data(ids));
202 if (error < 0) {
203 exodus_error(__LINE__);
204 }
205 }
206 }
207
208 template <typename INT> bool is_sequential(std::vector<INT> &map)
209 {
210 for (size_t i = 0; i < map.size(); i++) {
211 if (map[i] != (INT)i + 1) {
212 return false;
213 }
214 }
215 return true;
216 }
217
218 // SEE: http://lemire.me/blog/2017/04/10/removing-duplicates-from-lists-quickly
219 template <typename T> size_t unique(std::vector<T> &out)
220 {
221 if (out.empty())
222 return 0;
223 size_t i = 1;
224 size_t pos = 1;
225 T oldv = out[0];
226 for (; i < out.size(); ++i) {
227 T newv = out[i];
228 out[pos] = newv;
229 pos += (newv != oldv);
230 oldv = newv;
231 }
232 return pos;
233 }
234
235 template <typename T> void uniquify(std::vector<T> &vec)
236 {
237
238 #if USE_STD_SORT
239 std::sort(vec.begin(), vec.end());
240 #else
241 pdqsort(vec.begin(), vec.end());
242 #endif
243 vec.resize(unique(vec));
244 vec.shrink_to_fit();
245 }
246
247 void compress_white_space(char *str);
248 void add_info_record(char *info_record, int size);
249 void put_global_info(const Excn::Mesh &global);
250 void get_put_qa(int id, int id_out);
251 void get_put_coordinate_names(int in, int out, int dimensionality);
252 void get_put_assemblies(int in, int out, Excn::Mesh &global);
253
254 template <typename T> void get_put_coordinate_frames(int id, int id_out);
255
256 template <typename T, typename INT>
257 void get_put_coordinates(Excn::Mesh &global, int part_count, std::vector<Excn::Mesh> &local_mesh,
258 const std::vector<std::vector<INT>> &local_node_to_global);
259
260 template <typename T, typename INT>
261 void get_coordinates(int id, int dimensionality, size_t num_nodes,
262 const std::vector<std::vector<INT>> &local_node_to_global, int proc,
263 std::vector<T> &x, std::vector<T> &y, std::vector<T> &z);
264
265 template <typename INT>
266 void get_put_nodal_communication_map(int part_count,
267 const std::vector<std::vector<INT>> &local_node_to_global,
268 const std::vector<int> &processor_map, int output_processor);
269
270 StringVector get_exodus_variable_names(int id, ex_entity_type elType, int var_count);
271
272 template <typename T>
273 void filter_truth_table(int id, Excn::Mesh &global, std::vector<T> &glob_blocks,
274 Excn::Variables &vars, const Excn::StringIdVector &variable_names);
275
276 template <typename T>
277 void get_truth_table(Excn::Mesh &global, std::vector<T> &glob_blocks,
278 std::vector<Excn::Mesh> &local, Excn::Variables &vars, int debug);
279
280 template <typename U>
281 void create_output_truth_table(const Excn::Mesh &global, std::vector<U> &global_sets,
282 Excn::Variables &vars, std::vector<int> &truth_table);
283
284 template <typename T, typename U, typename INT>
285 void read_write_master_values(Excn::Variables &vars, const Excn::Mesh &global,
286 std::vector<U> &global_sets, std::vector<Excn::Mesh> &local_mesh,
287 std::vector<std::vector<U>> &local_sets,
288 MasterValueVector<T> &master_values, std::vector<T> &values,
289 int part_count, int time_step, int time_step_out,
290 const std::vector<std::vector<INT>> &local_entity_to_global);
291
292 void get_variable_params(int id, Excn::Variables &vars,
293 const Excn::StringIdVector &variable_list);
294
295 void get_put_variable_names(int id, int out, Excn::Variables &vars,
296 Excn::SystemInterface &interFace);
297
298 template <typename INT>
299 void build_reverse_element_map(std::vector<std::vector<INT>> &local_element_to_global,
300 const std::vector<Excn::Mesh> &local_mesh,
301 std::vector<std::vector<Excn::Block>> &blocks,
302 std::vector<Excn::Block> &glob_blocks, Excn::Mesh *global,
303 int part_count, std::vector<INT> &global_element_map,
304 bool map_ids);
305
306 template <typename T, typename INT>
307 void get_nodesets(int part_count, size_t total_node_count,
308 const std::vector<std::vector<INT>> &local_node_to_global,
309 std::vector<std::vector<Excn::NodeSet<INT>>> &nodesets,
310 std::vector<Excn::NodeSet<INT>> &glob_sets);
311
312 template <typename INT>
313 void build_reverse_node_map(std::vector<std::vector<INT>> &local_node_to_global,
314 const std::vector<Excn::Mesh> &local_mesh, Excn::Mesh *global,
315 int part_count, std::vector<INT> &global_node_map);
316
317 void get_element_blocks(int part_count, const std::vector<Excn::Mesh> &local_mesh,
318 const Excn::Mesh &global, std::vector<std::vector<Excn::Block>> &blocks,
319 std::vector<Excn::Block> &glob_blocks);
320
321 template <typename T, typename INT>
322 void put_element_blocks(int part_count, int start_part,
323 std::vector<std::vector<Excn::Block>> &blocks,
324 std::vector<Excn::Block> &glob_blocks,
325 const std::vector<std::vector<INT>> &local_node_to_global,
326 const std::vector<std::vector<INT>> &local_element_to_global);
327
328 template <typename T, typename INT>
329 void put_element_blocks(int part_count, int start_part,
330 std::vector<std::vector<Excn::Block>> &blocks,
331 std::vector<Excn::Block> &glob_blocks,
332 const std::vector<std::vector<INT>> &local_node_to_global);
333
334 template <typename INT>
335 void get_edgeblocks(int part_count, const std::vector<Excn::Mesh> &local_mesh,
336 const Excn::Mesh &global,
337 std::vector<std::vector<Excn::EdgeBlock<INT>>> &edgeblocks,
338 std::vector<Excn::EdgeBlock<INT>> &glob_edgeblocks);
339 template <typename T, typename INT>
340 void put_edgeblocks(int part_count, int start_part,
341 std::vector<std::vector<Excn::EdgeBlock<INT>>> &edgeblocks,
342 std::vector<Excn::EdgeBlock<INT>> &glob_edgeblocks,
343 const std::vector<std::vector<INT>> &local_node_to_global,
344 const std::vector<std::vector<INT>> &local_element_to_global);
345
346 template <typename T, typename INT>
347 void put_edgeblocks(int part_count, int start_part,
348 std::vector<std::vector<Excn::EdgeBlock<INT>>> &edgeblocks,
349 std::vector<Excn::EdgeBlock<INT>> &glob_edgeblocks,
350 const std::vector<std::vector<INT>> &local_node_to_global);
351
352 template <typename INT>
353 void build_reverse_edge_map(std::vector<std::vector<INT>> &local_edge_to_global,
354 const std::vector<Excn::Mesh> &local_mesh,
355 std::vector<std::vector<Excn::EdgeBlock<INT>>> &edgeblocks,
356 std::vector<Excn::EdgeBlock<INT>> &glob_edgeblocks,
357 Excn::Mesh *global, int part_count, std::vector<INT> &global_edge_map,
358 bool map_ids);
359
360 template <typename INT>
361 void get_faceblocks(int part_count, const std::vector<Excn::Mesh> &local_mesh,
362 const Excn::Mesh &global,
363 std::vector<std::vector<Excn::FaceBlock<INT>>> &faceblocks,
364 std::vector<Excn::FaceBlock<INT>> &glob_faceblocks);
365 template <typename T, typename INT>
366 void put_faceblocks(int part_count, int start_part,
367 std::vector<std::vector<Excn::FaceBlock<INT>>> &faceblocks,
368 std::vector<Excn::FaceBlock<INT>> &glob_faceblocks,
369 const std::vector<std::vector<INT>> &local_node_to_global,
370 const std::vector<std::vector<INT>> &local_element_to_global);
371
372 template <typename T, typename INT>
373 void put_faceblocks(int part_count, int start_part,
374 std::vector<std::vector<Excn::FaceBlock<INT>>> &faceblocks,
375 std::vector<Excn::FaceBlock<INT>> &glob_faceblocks,
376 const std::vector<std::vector<INT>> &local_node_to_global);
377
378 template <typename INT>
379 void build_reverse_face_map(std::vector<std::vector<INT>> &local_face_to_global,
380 const std::vector<Excn::Mesh> &local_mesh,
381 std::vector<std::vector<Excn::FaceBlock<INT>>> &faceblocks,
382 std::vector<Excn::FaceBlock<INT>> &glob_faceblocks,
383 Excn::Mesh *global, int part_count, std::vector<INT> &global_face_map,
384 bool map_ids);
385
386 template <typename INT> void put_nodesets(std::vector<Excn::NodeSet<INT>> &glob_sets);
387
388 template <typename INT>
389 void get_sideset_metadata(int part_count, std::vector<std::vector<Excn::SideSet<INT>>> &sets,
390 std::vector<Excn::SideSet<INT>> &glob_ssets);
391
392 template <typename INT>
393 void
394 get_put_sidesets(int part_count, const std::vector<std::vector<INT>> &local_element_to_global,
395 std::vector<std::vector<Excn::SideSet<INT>>> &sets,
396 std::vector<Excn::SideSet<INT>> &glob_ssets, Excn::SystemInterface &interFace);
397
398 template <typename INT>
399 void add_processor_map(int id_out, int part_count, int start_part, const Excn::Mesh &global,
400 std::vector<std::vector<Excn::Block>> &blocks,
401 const std::vector<Excn::Block> &glob_blocks,
402 const std::vector<std::vector<INT>> &local_element_to_global);
403
404 template <typename T, typename INT>
405 void add_processor_variable(int id_out, int part_count, int start_part, const Excn::Mesh &global,
406 std::vector<std::vector<Excn::Block>> &blocks,
407 const std::vector<Excn::Block> &glob_blocks,
408 const std::vector<std::vector<INT>> &local_element_to_global,
409 int step, int variable, std::vector<T> &proc);
410
411 template <typename INT>
412 size_t find_max_entity_count(int part_count, std::vector<Excn::Mesh> &local_mesh,
413 const Excn::Mesh &global,
414 std::vector<std::vector<Excn::Block>> &blocks,
415 std::vector<std::vector<Excn::NodeSet<INT>>> &nodesets,
416 std::vector<std::vector<Excn::SideSet<INT>>> &sidesets,
417 std::vector<std::vector<Excn::EdgeBlock<INT>>> &edgeblocks,
418 std::vector<std::vector<Excn::FaceBlock<INT>>> &faceblocks);
419
420 template <typename INT>
421 size_t find_max_global_entity_count(const Excn::Mesh &global, std::vector<Excn::Block> &blocks,
422 std::vector<Excn::NodeSet<INT>> &nodesets,
423 std::vector<Excn::SideSet<INT>> &sidesets,
424 std::vector<Excn::EdgeBlock<INT>> &edgeblocks,
425 std::vector<Excn::FaceBlock<INT>> &faceblocks);
426
427 } // namespace
428
429 using namespace Excn;
430
431 int main(int argc, char *argv[])
432 {
433 mpi my_mpi(argc, argv);
434 rank = my_mpi.rank;
435
436 try {
437 time_t begin_time = std::time(nullptr);
438 SystemInterface::show_version(rank);
439 if (rank == 0) {
440 #if ENABLE_PARALLEL_EPU
441 fmt::print("\tParallel Capability Enabled.\n");
442 #else
443 fmt::print("\tParallel Capability Not Enabled.\n");
444 #endif
445 }
446
447 SystemInterface interFace(rank);
448 bool execute = interFace.parse_options(argc, argv);
449
450 if (!execute) {
451 return EXIT_SUCCESS;
452 }
453
454 // Debug Options: (can be or'd together)
455 // 1 -- time stamp
456 // 2 -- check nodal variable consistency
457 // 4 -- Element Blocks
458 // 8 -- Nodes
459 // 16 -- Sidesets
460 // 32 -- Nodesets
461 // 64 -- Edge Blocks
462 // 128 -- Face Blocks
463 // 256 -- exodus verbose.
464 // 512 -- Check consistent global field values between processors
465 debug_level = interFace.debug();
466
467 if ((debug_level & 256) != 0U) {
468 ex_opts(EX_VERBOSE | EX_DEBUG);
469 }
470 else {
471 ex_opts(0);
472 }
473
474 int start_part = interFace.start_part();
475 int processor_count = interFace.processor_count();
476
477 int part_count = interFace.part_count();
478 if (part_count <= 1) {
479 fmt::print("INFO: Only one processor or part, no concatenation needed.\n");
480 return EXIT_SUCCESS;
481 }
482
483 int error = 0;
484 if (my_mpi.epu_proc_count > 1) {
485 interFace.subcycle(my_mpi.epu_proc_count);
486
487 int per_proc = processor_count / my_mpi.epu_proc_count;
488 int extra = processor_count % my_mpi.epu_proc_count;
489
490 part_count = per_proc + (rank < extra ? 1 : 0);
491
492 if (rank < extra) {
493 start_part = (per_proc + 1) * rank;
494 }
495 else {
496 start_part = (per_proc + 1) * extra + per_proc * (rank - extra);
497 }
498
499 SMART_ASSERT(start_part + part_count <= processor_count);
500
501 ExodusFile::initialize(interFace, start_part, part_count, rank, false);
502
503 if (ExodusFile::io_word_size() == 4) { // Reals are floats
504 if (interFace.int64()) {
505 error = epu<float, int64_t>(interFace, start_part, part_count, rank);
506 }
507 else {
508 error = epu<float, int>(interFace, start_part, part_count, rank);
509 }
510 }
511 else { // Reals are doubles
512 if (interFace.int64()) {
513 error = epu<double, int64_t>(interFace, start_part, part_count, rank);
514 }
515 else {
516 error = epu<double, int>(interFace, start_part, part_count, rank);
517 }
518 }
519
520 ExodusFile::close_all();
521 #if ENABLE_PARALLEL_EPU
522 MPI_Barrier(MPI_COMM_WORLD); // CHECK: ALLOW MPI_COMM_WORLD
523 #endif
524 }
525 else {
526 int max_open_file = open_file_limit() - 1; // -1 for output exodus file.
527
528 // Only used to test the auto subcycle without requiring thousands of files...
529 if (interFace.max_open_files() > 0) {
530 max_open_file = interFace.max_open_files();
531 }
532
533 if (interFace.is_auto() && interFace.subcycle() < 0 && processor_count > max_open_file &&
534 part_count == processor_count && interFace.cycle() == -1) {
535 // Rule of thumb -- number of subcycles = cube_root(processor_count);
536 // if that value > max_open_file, then use square root.
537 // if that is still too large, just do no subcycles... and implement
538 // a recursive subcycling capability at some point...
539 int sub_cycle_count = (int)(std::pow(processor_count, 1.0 / 3) + 0.9);
540 if (((processor_count + sub_cycle_count - 1) / sub_cycle_count) > max_open_file) {
541 sub_cycle_count = (int)std::sqrt(processor_count);
542 }
543
544 if (((processor_count + sub_cycle_count - 1) / sub_cycle_count) < max_open_file) {
545 interFace.subcycle(sub_cycle_count);
546 if (rank == 0) {
547 fmt::print("\tAutomatically activating subcyle mode\n\tNumber of processors ({}) "
548 "exceeds open file limit ({}).\n"
549 "\tUsing --subcycle={}\n\n",
550 processor_count, max_open_file, sub_cycle_count);
551 }
552 interFace.subcycle_join(true);
553 }
554 }
555
556 int cycle = interFace.cycle();
557 if (interFace.subcycle() >= 0) {
558 start_part = 0;
559 int cycles = interFace.subcycle();
560 if (cycles > 0) {
561 // use the specified number of cycles...
562 part_count = (processor_count + cycles - 1) / cycles;
563 if (cycle >= 0) {
564 start_part = cycle * part_count;
565 }
566 }
567
568 // Sanity check...
569 if (part_count < 1) {
570 throw std::runtime_error(
571 "ERROR: (EPU) The subcycle specification results in less than 1 part per "
572 "cycle which is not allowed.\n");
573 }
574 interFace.subcycle((processor_count + part_count - 1) / part_count);
575
576 if (start_part + part_count > processor_count) {
577 part_count = processor_count - start_part;
578 }
579 }
580
581 if (cycle < 0) {
582 cycle = 0;
583 }
584 while (start_part < processor_count) {
585
586 if (start_part + part_count > processor_count) {
587 part_count = processor_count - start_part;
588 }
589
590 SMART_ASSERT(part_count > 0);
591 SMART_ASSERT(start_part + part_count <= processor_count);
592
593 ExodusFile::initialize(interFace, start_part, part_count, cycle, false);
594
595 if (ExodusFile::io_word_size() == 4) { // Reals are floats
596 if (interFace.int64()) {
597 error = epu<float, int64_t>(interFace, start_part, part_count, cycle++);
598 }
599 else {
600 error = epu<float, int>(interFace, start_part, part_count, cycle++);
601 }
602 }
603 else { // Reals are doubles
604 if (interFace.int64()) {
605 error = epu<double, int64_t>(interFace, start_part, part_count, cycle++);
606 }
607 else {
608 error = epu<double, int>(interFace, start_part, part_count, cycle++);
609 }
610 }
611
612 start_part += part_count;
613 ExodusFile::close_all();
614 if (interFace.subcycle() < 0 || (interFace.subcycle() > 0 && interFace.cycle() >= 0)) {
615 break;
616 }
617 }
618 }
619
620 if (interFace.subcycle() > 0 && interFace.cycle() < 0 && interFace.subcycle_join() &&
621 rank == 0) {
622 // Now, join the subcycled parts into a single file...
623 start_part = 0;
624 part_count = interFace.subcycle();
625 interFace.subcycle(0);
626 interFace.processor_count(part_count);
627 interFace.step_min(1);
628 interFace.step_max(INT_MAX);
629 interFace.step_interval(1);
630
631 ExodusFile::initialize(interFace, start_part, part_count, 0, true);
632 if (ExodusFile::io_word_size() == 4) { // Reals are floats
633 if (interFace.int64()) {
634 error = epu<float, int64_t>(interFace, start_part, part_count, 0);
635 }
636 else {
637 error = epu<float, int>(interFace, start_part, part_count, 0);
638 }
639 }
640 else { // Reals are doubles
641 if (interFace.int64()) {
642 error = epu<double, int64_t>(interFace, start_part, part_count, 0);
643 }
644 else {
645 error = epu<double, int>(interFace, start_part, part_count, 0);
646 }
647 }
648
649 bool delete_temp = error == 0 && !interFace.keep_temporary();
650 ExodusFile::handle_temporary_files(delete_temp);
651 }
652
653 if (error == 0 && interFace.remove_file_per_rank_files()) {
654 ExodusFile::unlink_input_files();
655 }
656
657 time_t end_time = std::time(nullptr);
658 if (rank == 0) {
659 add_to_log(argv[0], static_cast<int>(end_time - begin_time));
660 }
661 return error;
662 }
663 catch (std::exception &e) {
664 fmt::print(stderr, "{}\n", e.what());
665 return EXIT_FAILURE;
666 }
667 }
668
669 template <typename T, typename INT>
670 int epu(SystemInterface &interFace, int start_part, int part_count, int cycle)
671 {
672 double execution_time = seacas_timer();
673 SMART_ASSERT(sizeof(T) == ExodusFile::io_word_size());
674
675 if (rank == 0) {
676 fmt::print("\nIO Word sizes: {} bytes floating point and {} bytes integer.\n", sizeof(T),
677 sizeof(INT));
678 }
679 std::vector<char> mytitle(MAX_LINE_LENGTH + 1, '\0');
680
681 Mesh global;
682
683 // contains the global node information from each file/processor
684 std::vector<std::vector<INT>> local_node_to_global(part_count);
685
686 // contains the global element information from each file/processor
687 std::vector<std::vector<INT>> local_element_to_global(part_count);
688
689 // contains the global edge information from each file/processor
690 std::vector<std::vector<INT>> local_edge_to_global(part_count);
691
692 // contains the global face information from each file/processor
693 std::vector<std::vector<INT>> local_face_to_global(part_count);
694
695 std::vector<Mesh> local_mesh(part_count);
696
697 // ******************************************************************
698 // 1. Read global info
699
700 bool first_change_set = true;
701 int error = 0;
702
703 int cs_count = ExodusFile::get_change_set_count();
704 int cs_start = (cs_count > 0 ? 1 : 0);
705 if (interFace.selected_change_set() > 0) {
706 cs_start = interFace.selected_change_set();
707 cs_count = cs_start;
708 }
709 for (int cs = cs_start; cs <= cs_count; cs++) {
710 ExodusFile::set_active_change_set(cs);
711
712 LOG(fmt::format("\n\n**** PROCESSING CHANGE SET {} of {} ****\n",
713 ExodusFile::get_active_change_set(), ExodusFile::get_change_set_count()));
714
715 LOG("\n**** READ LOCAL (GLOBAL) INFO ****\n");
716 std::string title0;
717
718 // EPU assumes IDS are always passed through the API as 64-bit ints.
719 SMART_ASSERT(ex_int64_status(ExodusFile(0)) & EX_IDS_INT64_API);
720
721 if (sizeof(INT) == 8) {
722 SMART_ASSERT((ex_int64_status(ExodusFile(0)) & EX_BULK_INT64_API) != 0);
723 SMART_ASSERT((ex_int64_status(ExodusFile(0)) & EX_MAPS_INT64_API) != 0);
724 }
725 else {
726 SMART_ASSERT(sizeof(INT) == 4);
727 SMART_ASSERT((ex_int64_status(ExodusFile(0)) & EX_BULK_INT64_API) == 0);
728 SMART_ASSERT((ex_int64_status(ExodusFile(0)) & EX_MAPS_INT64_API) == 0);
729 }
730
731 // If there are any processors with zero nodes, then
732 // that node won't have any nodal variables defined. We need to
733 // find the first processor which has a non-zero node count to use
734 // when we look for the nodal variable count.
735 int64_t non_zero_node_count = -1;
736 for (int p = 0; p < part_count; p++) {
737 ex_init_params exodus{};
738 error = ex_get_init_ext(ExodusFile(p), &exodus);
739 if (error < 0) {
740 exodus_error(__LINE__);
741 }
742
743 local_mesh[p].dimensionality = exodus.num_dim;
744 local_mesh[p].nodeCount = exodus.num_nodes;
745 local_mesh[p].elementCount = exodus.num_elem;
746 local_mesh[p].edgeCount = exodus.num_edge;
747 local_mesh[p].faceCount = exodus.num_face;
748 local_mesh[p].blockCount = exodus.num_elem_blk;
749 local_mesh[p].nodesetCount = exodus.num_node_sets;
750 local_mesh[p].sidesetCount = exodus.num_side_sets;
751 local_mesh[p].assemblyCount = exodus.num_assembly;
752 local_mesh[p].edgeBlockCount = exodus.num_edge_blk;
753 local_mesh[p].faceBlockCount = exodus.num_face_blk;
754 local_mesh[p].title = exodus.title;
755
756 if (local_mesh[p].nodeCount > 0 && non_zero_node_count == -1) {
757 non_zero_node_count = p;
758 }
759
760 if (p == 0) {
761 global.title = Data(mytitle);
762 global.dimensionality = local_mesh[p].dimensionality;
763 global.blockCount = local_mesh[p].count(Excn::ObjectType::EBLK);
764 global.nodesetCount = local_mesh[p].count(Excn::ObjectType::NSET);
765 global.sidesetCount = local_mesh[p].count(Excn::ObjectType::SSET);
766 global.assemblyCount = local_mesh[p].count(Excn::ObjectType::ASSM);
767 global.edgeBlockCount = local_mesh[p].count(Excn::ObjectType::EDBLK);
768 global.faceBlockCount = local_mesh[p].count(Excn::ObjectType::FABLK);
769 }
770 else {
771 SMART_ASSERT(global.dimensionality == local_mesh[p].dimensionality);
772 SMART_ASSERT(global.count(Excn::ObjectType::EBLK) ==
773 local_mesh[p].count(Excn::ObjectType::EBLK));
774 SMART_ASSERT(global.count(Excn::ObjectType::ASSM) ==
775 local_mesh[p].count(Excn::ObjectType::ASSM));
776 if (!interFace.omit_nodesets()) {
777 SMART_ASSERT(global.count(Excn::ObjectType::NSET) ==
778 local_mesh[p].count(Excn::ObjectType::NSET));
779 }
780 if (!interFace.omit_sidesets()) {
781 SMART_ASSERT(global.count(Excn::ObjectType::SSET) ==
782 local_mesh[p].count(Excn::ObjectType::SSET));
783 }
784 if (!interFace.omit_edgeblocks()) {
785 SMART_ASSERT(global.count(Excn::ObjectType::EDBLK) ==
786 local_mesh[p].count(Excn::ObjectType::EDBLK));
787 }
788 if (!interFace.omit_faceblocks()) {
789 SMART_ASSERT(global.count(Excn::ObjectType::FABLK) ==
790 local_mesh[p].count(Excn::ObjectType::FABLK));
791 }
792 }
793
794 local_node_to_global[p].resize(local_mesh[p].nodeCount);
795 local_element_to_global[p].resize(local_mesh[p].elementCount);
796 local_edge_to_global[p].resize(local_mesh[p].edgeCount);
797 local_face_to_global[p].resize(local_mesh[p].faceCount);
798
799 // sum required data
800 // note that num_blocks is the same for every processor
801 global.elementCount += local_mesh[p].elementCount;
802 global.edgeCount += local_mesh[p].edgeCount;
803 global.faceCount += local_mesh[p].faceCount;
804
805 } // end for (p=0..part_count)
806
807 if (non_zero_node_count == -1) {
808 non_zero_node_count = 0; // No nodes on entire model...
809 }
810
811 if (interFace.omit_edgeblocks()) {
812 global.edgeBlockCount = 0;
813 }
814
815 if (interFace.omit_faceblocks()) {
816 global.faceBlockCount = 0;
817 }
818
819 if (interFace.omit_nodesets()) {
820 global.nodesetCount = 0;
821 }
822
823 if (interFace.omit_sidesets()) {
824 global.sidesetCount = 0;
825 }
826
827 // Need these throughout run, so declare outside of this block...
828 std::vector<Block> glob_blocks(global.count(Excn::ObjectType::EBLK));
829 std::vector<std::vector<Block>> blocks(part_count);
830
831 std::vector<EdgeBlock<INT>> glob_edgeblocks(global.count(Excn::ObjectType::EDBLK));
832 std::vector<std::vector<EdgeBlock<INT>>> edgeblocks(part_count);
833
834 std::vector<FaceBlock<INT>> glob_faceblocks(global.count(Excn::ObjectType::FABLK));
835 std::vector<std::vector<FaceBlock<INT>>> faceblocks(part_count);
836
837 std::vector<SideSet<INT>> glob_ssets;
838 std::vector<std::vector<SideSet<INT>>> sidesets(part_count);
839
840 std::vector<NodeSet<INT>> glob_nsets;
841 std::vector<std::vector<NodeSet<INT>>> nodesets(part_count);
842 {
843 // Now, build the reverse global node map which permits access of the
844 // local id given the global id.
845 std::vector<INT> global_node_map;
846 build_reverse_node_map(local_node_to_global, local_mesh, &global, part_count,
847 global_node_map);
848
849 LOG("Finished reading/writing Global Info\n");
850 if (interFace.output_shared_nodes()) {
851 // Get list of all shared nodes...
852 std::vector<std::vector<INT>> shared(part_count);
853 std::vector<int> num_shared(global_node_map.size());
854
855 for (auto &part : shared) {
856 part.resize(global_node_map.size());
857 }
858
859 for (int pc = 0; pc < part_count; pc++) {
860 size_t node_count = local_mesh[pc].nodeCount;
861 for (size_t i = 0; i < node_count; i++) {
862 INT gloc = local_node_to_global[pc][i];
863 shared[pc][gloc] = i + 1;
864 num_shared[gloc]++;
865 }
866 }
867
868 if (rank == 0) {
869 fmt::print("Node Sharing information: (Part:Local Node Id)\n");
870 for (size_t i = 0; i < global_node_map.size(); i++) {
871 if (num_shared[i] > 1) {
872 fmt::print("Global Node {}:", fmt::group_digits(i + 1));
873 for (int pc = 0; pc < part_count; pc++) {
874 if (shared[pc][i] >= 1) {
875 fmt::print("\t{}:{}", pc, fmt::group_digits(shared[pc][i]));
876 }
877 }
878 fmt::print("\n");
879 }
880 }
881 }
882 }
883
884 // ****************************************************************************
885 // Get Block information including element attributes
886 // must check for zero length blocks
887 get_element_blocks(part_count, local_mesh, global, blocks, glob_blocks);
888
889 bool map_element_ids = interFace.map_element_ids();
890 if (interFace.subcycle() >= 0) {
891 map_element_ids = false;
892 }
893 std::vector<INT> global_element_map(global.elementCount);
894 build_reverse_element_map(local_element_to_global, local_mesh, blocks, glob_blocks, &global,
895 part_count, global_element_map, map_element_ids);
896
897 get_edgeblocks(part_count, local_mesh, global, edgeblocks, glob_edgeblocks);
898 if (glob_edgeblocks.size() > 0) {
899 bool map_edge_ids = interFace.map_edge_ids();
900 if (interFace.subcycle() >= 0) {
901 map_edge_ids = false;
902 }
903 std::vector<INT> global_edge_map(global.edgeCount);
904 build_reverse_edge_map(local_edge_to_global, local_mesh, edgeblocks, glob_edgeblocks,
905 &global, part_count, global_edge_map, map_edge_ids);
906 }
907
908 get_faceblocks(part_count, local_mesh, global, faceblocks, glob_faceblocks);
909 if (glob_faceblocks.size() > 0) {
910 bool map_face_ids = interFace.map_face_ids();
911 if (interFace.subcycle() >= 0) {
912 map_face_ids = false;
913 }
914 std::vector<INT> global_face_map(global.faceCount);
915 build_reverse_face_map(local_face_to_global, local_mesh, faceblocks, glob_faceblocks,
916 &global, part_count, global_face_map, map_face_ids);
917 }
918
919 //
920 // NOTE: Node set/side set information can be different for each processor
921 /************************************************************************/
922 // Get Side sets
923 if (!interFace.omit_sidesets()) {
924 LOG("\n**** GET SIDE SETS *****\n");
925 get_sideset_metadata(part_count, sidesets, glob_ssets);
926 if (global.count(Excn::ObjectType::SSET) != glob_ssets.size()) {
927 fmt::print("\nWARNING: Invalid sidesets will not be written to output database.\n");
928 global.sidesetCount = glob_ssets.size();
929 }
930 }
931
932 /************************************************************************/
933 // Get Node sets
934 if (!interFace.omit_nodesets()) {
935 LOG("\n**** GET NODE SETS *****\n");
936 get_nodesets<T>(part_count, global.nodeCount, local_node_to_global, nodesets, glob_nsets);
937 if (global.count(Excn::ObjectType::NSET) != glob_nsets.size()) {
938 fmt::print("\nWARNING: Invalid nodesets will not be written to output database.\n");
939 global.nodesetCount = glob_nsets.size();
940 }
941 }
942
943 /************************************************************************/
944 // Start writing the output file...
945
946 LOG("\n**** BEGIN WRITING OUTPUT FILE *****\n");
947 CommunicationMetaData comm_data;
948
949 if (!interFace.int64()) {
950 int64_t twoBill = 1;
951 twoBill <<= 31;
952 int64_t fourBill = 1;
953 fourBill <<= 32;
954 // Check whether output mesh requires 64-bit integers...
955 if (global.nodeCount >= twoBill || global.elementCount >= twoBill) {
956 throw std::runtime_error(
957 "\n\nERROR: (EPU) Output file requires 64-bit integers. You must "
958 "rerun epu with the -64 option.\n\n");
959 }
960
961 if (!interFace.use_netcdf4() && !interFace.use_netcdf5()) {
962 // Check size required to store coordinates and connectivity
963 if (global.nodeCount * 8 >= fourBill) {
964 fmt::print(stderr, "\nINFO: Output file requires NetCDF-4 format or NetCDF-5. Setting "
965 "NetCDF-4 automatically.\n\n");
966 interFace.set_use_netcdf4();
967 }
968
969 for (const auto &block : glob_blocks) {
970 int64_t element_count = block.entity_count();
971 int64_t nnpe = block.nodesPerElement;
972 if (element_count * nnpe * 4 >= fourBill) {
973 fmt::print(stderr,
974 "\nINFO: Output file requires NetCDF-4 format or NetCDF-5. Setting "
975 "NetCDF-4 automatically.\n\n");
976 interFace.set_use_netcdf4();
977 break;
978 }
979 }
980 }
981 }
982
983 if (first_change_set) {
984 ExodusFile::create_output(interFace, cycle);
985 first_change_set = false;
986 }
987
988 // EPU assumes IDS are always passed through the API as 64-bit ints.
989 SMART_ASSERT(ex_int64_status(ExodusFile::output()) & EX_IDS_INT64_API);
990
991 if (sizeof(INT) == 8) {
992 SMART_ASSERT((ex_int64_status(ExodusFile::output()) & EX_BULK_INT64_API) != 0);
993 SMART_ASSERT((ex_int64_status(ExodusFile::output()) & EX_MAPS_INT64_API) != 0);
994 }
995 else {
996 SMART_ASSERT(sizeof(INT) == 4);
997 SMART_ASSERT((ex_int64_status(ExodusFile::output()) & EX_BULK_INT64_API) == 0);
998 SMART_ASSERT((ex_int64_status(ExodusFile::output()) & EX_MAPS_INT64_API) == 0);
999 }
1000
1001 // Define metadata for model....
1002 put_global_info(global);
1003 get_put_coordinate_frames<T>(ExodusFile(0), ExodusFile::output());
1004
1005 Internals<INT> exodus(ExodusFile::output(), ExodusFile::max_name_length(),
1006 ExodusFile::get_active_change_set());
1007
1008 if (interFace.append()) {
1009 bool matches = exodus.check_meta_data(global, glob_blocks, glob_nsets, glob_ssets,
1010 glob_edgeblocks, glob_faceblocks, comm_data);
1011 if (!matches) {
1012 throw std::runtime_error("\n\nERROR: (EPU) Current mesh dimensions do not match "
1013 "the mesh dimensions in the file being appended to.\n\n");
1014 }
1015 }
1016 else {
1017 global.needNodeMap = !is_sequential(global_node_map);
1018 global.needElementMap = !is_sequential(global_element_map);
1019
1020 exodus.write_meta_data(global, glob_blocks, glob_nsets, glob_ssets, glob_edgeblocks,
1021 glob_faceblocks, comm_data);
1022
1023 if (ExodusFile::get_active_change_set() <= 1) {
1024 int id_out = ExodusFile::output();
1025 get_put_qa(ExodusFile(0), id_out);
1026 }
1027
1028 get_put_assemblies(ExodusFile(0), ExodusFile::output(), global);
1029
1030 if (interFace.add_processor_id_map()) {
1031 add_processor_map(ExodusFile::output(), part_count, start_part, global, blocks,
1032 glob_blocks, local_element_to_global);
1033 }
1034
1035 // Output bulk mesh data....
1036 put_nodesets(glob_nsets);
1037
1038 // c.2. Write Global Node Number Map
1039 if (global.needNodeMap) {
1040 LOG("Writing global node number map...\n");
1041 error = ex_put_id_map(ExodusFile::output(), EX_NODE_MAP, Data(global_node_map));
1042 if (error < 0) {
1043 exodus_error(__LINE__);
1044 }
1045 }
1046
1047 if (global.needElementMap) {
1048 LOG("Writing out master global elements information...\n");
1049 if (!global_element_map.empty()) {
1050 error = ex_put_id_map(ExodusFile::output(), EX_ELEM_MAP, Data(global_element_map));
1051 if (error < 0) {
1052 exodus_error(__LINE__);
1053 }
1054 }
1055 }
1056
1057 // Needed on glory writing to Lustre or we end up with empty maps...
1058 ex_update(ExodusFile::output());
1059
1060 if (interFace.map_element_ids()) {
1061 put_element_blocks<T>(part_count, start_part, blocks, glob_blocks, local_node_to_global,
1062 local_element_to_global);
1063 put_edgeblocks<T>(part_count, start_part, edgeblocks, glob_edgeblocks,
1064 local_node_to_global, local_edge_to_global);
1065 put_faceblocks<T>(part_count, start_part, faceblocks, glob_faceblocks,
1066 local_node_to_global, local_face_to_global);
1067 }
1068 else {
1069 put_element_blocks<T>(part_count, start_part, blocks, glob_blocks, local_node_to_global);
1070 put_edgeblocks<T>(part_count, start_part, edgeblocks, glob_edgeblocks,
1071 local_node_to_global);
1072 put_faceblocks<T>(part_count, start_part, faceblocks, glob_faceblocks,
1073 local_node_to_global);
1074 }
1075 }
1076
1077 get_put_sidesets(part_count, local_element_to_global, sidesets, glob_ssets, interFace);
1078 }
1079 // ************************************************************************
1080 // 2. Get Coordinate Info.
1081 if (!interFace.append()) {
1082 LOG("\n\n**** GET COORDINATE INFO ****\n");
1083 get_put_coordinates<T>(global, part_count, local_mesh, local_node_to_global);
1084 LOG("Wrote coordinate information...\n");
1085 }
1086
1087 if (interFace.add_nodal_communication_map() && interFace.subcycle() > 0) {
1088 LOG("\n\n**** GET NODAL COMMUNICATION MAP INFO ****\n");
1089 // Need a mapping from processors in the original mesh parts to
1090 // processors in the subcycle output...
1091 auto proc_count = interFace.processor_count();
1092
1093 std::vector<int> processor_map(proc_count);
1094 auto orig_part_count = interFace.part_count();
1095 for (int i = 0; i < proc_count; i++) {
1096 processor_map[i] = i / orig_part_count;
1097 }
1098 get_put_nodal_communication_map(part_count, local_node_to_global, processor_map, cycle);
1099 LOG("Wrote nodal communication map information...\n");
1100 }
1101
1102 // ####################TRANSIENT DATA SECTION###########################
1103 // ***********************************************************************
1104 // 9. Get Variable Information and names
1105 LOG("\n**** GET VARIABLE INFORMATION AND NAMES ****\n");
1106
1107 // I. read number of variables for each type.
1108 // NOTE: it is assumed that every processor has the same global, nodal,
1109 // and element lists
1110
1111 Variables global_vars(Excn::ObjectType::GLOBAL);
1112 Variables nodal_vars(Excn::ObjectType::NODE);
1113 Variables element_vars(Excn::ObjectType::EBLK);
1114 Variables nodeset_vars(Excn::ObjectType::NSET);
1115 Variables sideset_vars(Excn::ObjectType::SSET);
1116 Variables edgeblock_vars(Excn::ObjectType::EDBLK);
1117 Variables faceblock_vars(Excn::ObjectType::FABLK);
1118
1119 element_vars.addProcessorId = interFace.add_processor_id_field();
1120
1121 {
1122 ExodusFile id(non_zero_node_count);
1123
1124 get_variable_params(id, global_vars, interFace.global_var_names());
1125 get_variable_params(id, nodal_vars, interFace.node_var_names());
1126 get_variable_params(id, element_vars, interFace.elem_var_names());
1127 if (!interFace.omit_nodesets()) {
1128 get_variable_params(id, nodeset_vars, interFace.nset_var_names());
1129 }
1130 if (!interFace.omit_sidesets()) {
1131 get_variable_params(id, sideset_vars, interFace.sset_var_names());
1132 }
1133
1134 if (!interFace.omit_edgeblocks()) {
1135 get_variable_params(id, edgeblock_vars, interFace.edblk_var_names());
1136 }
1137
1138 if (!interFace.omit_faceblocks()) {
1139 get_variable_params(id, faceblock_vars, interFace.fablk_var_names());
1140 }
1141
1142 get_truth_table(global, glob_blocks, local_mesh, element_vars, 4);
1143 filter_truth_table(id, global, glob_blocks, element_vars, interFace.elem_var_names());
1144
1145 if (!interFace.omit_nodesets()) {
1146 get_truth_table(global, glob_nsets, local_mesh, nodeset_vars, 32);
1147 filter_truth_table(id, global, glob_nsets, nodeset_vars, interFace.nset_var_names());
1148 }
1149
1150 if (!interFace.omit_sidesets()) {
1151 get_truth_table(global, glob_ssets, local_mesh, sideset_vars, 16);
1152 filter_truth_table(id, global, glob_ssets, sideset_vars, interFace.sset_var_names());
1153 }
1154
1155 if (!interFace.omit_edgeblocks()) {
1156 get_truth_table(global, glob_edgeblocks, local_mesh, edgeblock_vars, 64);
1157 filter_truth_table(id, global, glob_edgeblocks, edgeblock_vars,
1158 interFace.edblk_var_names());
1159 }
1160
1161 if (!interFace.omit_faceblocks()) {
1162 get_truth_table(global, glob_faceblocks, local_mesh, faceblock_vars, 128);
1163 filter_truth_table(id, global, glob_faceblocks, faceblock_vars,
1164 interFace.fablk_var_names());
1165 }
1166 }
1167
1168 // There is a slightly tricky situation here. The truthTable block order
1169 // is based on the ordering of the blocks on the input databases.
1170 // These blocks may have been reordered on output to make the 'offset'
1171 // variables line up correctly when the element ids are mapped back to
1172 // the "overall global" order. There is not much problem since most
1173 // calls outputting block-related items pass the id and don't rely
1174 // on ordering. However, the truth table is one of the exceptions
1175 // and we need to reorder the truth table to match the output block
1176 // order. After this call, we can use the original ordering, so just
1177 // need a temporary vector here...
1178 if (global_vars.count(InOut::OUT) + nodal_vars.count(InOut::OUT) +
1179 element_vars.count(InOut::OUT) + nodeset_vars.count(InOut::OUT) +
1180 sideset_vars.count(InOut::OUT) + edgeblock_vars.count(InOut::OUT) +
1181 faceblock_vars.count(InOut::OUT) >
1182 0) {
1183
1184 std::vector<int> elem_truth_table(
1185 global.truthTable[static_cast<int>(Excn::ObjectType::EBLK)].size());
1186 create_output_truth_table(global, glob_blocks, element_vars, elem_truth_table);
1187
1188 if (!interFace.append()) {
1189 error = ex_put_all_var_param(
1190 ExodusFile::output(), global_vars.count(InOut::OUT), nodal_vars.count(InOut::OUT),
1191 element_vars.count(InOut::OUT), Data(elem_truth_table), nodeset_vars.count(InOut::OUT),
1192 Data(global.truthTable[static_cast<int>(Excn::ObjectType::NSET)]),
1193 sideset_vars.count(InOut::OUT),
1194 Data(global.truthTable[static_cast<int>(Excn::ObjectType::SSET)]));
1195 if (error < 0) {
1196 exodus_error(__LINE__);
1197 }
1198
1199 if (edgeblock_vars.count(InOut::OUT) > 0) {
1200 error = ex_put_variable_param(ExodusFile::output(), EX_EDGE_BLOCK,
1201 edgeblock_vars.count(InOut::OUT));
1202 if (error < 0) {
1203 exodus_error(__LINE__);
1204 }
1205 error = ex_put_truth_table(
1206 ExodusFile::output(), EX_EDGE_BLOCK, glob_edgeblocks.size(),
1207 edgeblock_vars.count(InOut::OUT),
1208 Data(global.truthTable[static_cast<int>(Excn::ObjectType::EDBLK)]));
1209 if (error < 0) {
1210 exodus_error(__LINE__);
1211 }
1212 }
1213
1214 if (faceblock_vars.count(InOut::OUT) > 0) {
1215 error = ex_put_variable_param(ExodusFile::output(), EX_FACE_BLOCK,
1216 faceblock_vars.count(InOut::OUT));
1217 if (error < 0) {
1218 exodus_error(__LINE__);
1219 }
1220 error = ex_put_truth_table(
1221 ExodusFile::output(), EX_FACE_BLOCK, glob_faceblocks.size(),
1222 faceblock_vars.count(InOut::OUT),
1223 Data(global.truthTable[static_cast<int>(Excn::ObjectType::FABLK)]));
1224 if (error < 0) {
1225 exodus_error(__LINE__);
1226 }
1227 }
1228 }
1229 }
1230
1231 // II. read/write the variable names
1232 {
1233 ExodusFile id(non_zero_node_count);
1234 get_put_variable_names(id, ExodusFile::output(), global_vars, interFace);
1235 get_put_variable_names(id, ExodusFile::output(), nodal_vars, interFace);
1236 get_put_variable_names(id, ExodusFile::output(), element_vars, interFace);
1237 if (!interFace.omit_nodesets()) {
1238 get_put_variable_names(id, ExodusFile::output(), nodeset_vars, interFace);
1239 }
1240 if (!interFace.omit_sidesets()) {
1241 get_put_variable_names(id, ExodusFile::output(), sideset_vars, interFace);
1242 }
1243 if (!interFace.omit_edgeblocks()) {
1244 get_put_variable_names(id, ExodusFile::output(), edgeblock_vars, interFace);
1245 }
1246 if (!interFace.omit_faceblocks()) {
1247 get_put_variable_names(id, ExodusFile::output(), faceblock_vars, interFace);
1248 }
1249 }
1250 if (!interFace.append()) {
1251 ex_update(ExodusFile::output());
1252 }
1253
1254 /**********************************************************************/
1255 // 10. Get Transient Data
1256 // This routine reads in a time dump from an EXODUSII file
1257
1258 int num_time_steps = 0;
1259
1260 LOG("\n**** GET TRANSIENT NODAL, GLOBAL, AND ELEMENT DATA VALUES ****\n");
1261 // Stage I: Get the number_of_time_steps information
1262
1263 bool differ = false;
1264 for (int p = 0; p < part_count; p++) {
1265 ExodusFile id(p);
1266
1267 int nts = ex_inquire_int(id, EX_INQ_TIME);
1268 if (p == 0) {
1269 num_time_steps = nts;
1270 }
1271 else {
1272 if (nts != num_time_steps) {
1273 differ = true;
1274 }
1275 num_time_steps = std::min(num_time_steps, nts);
1276 }
1277 }
1278 if (differ) {
1279 fmt::print(stderr,
1280 "\nWARNING: The number of time steps is not the same on all input databases.\n"
1281 " Using minimum count of {}\n\n",
1282 num_time_steps);
1283 }
1284 else {
1285 if (rank == 0) {
1286 fmt::print("\nNumber of time steps on input databases = {}\n\n", num_time_steps);
1287 }
1288 }
1289
1290 std::vector<T> global_values(global_vars.count(InOut::IN));
1291 std::vector<T> output_global_values(global_vars.count(InOut::OUT));
1292
1293 // TODO(gdsjaar): Handle variables via a class instead of 3-D array.
1294 // Determine maximum number of entities on any processor...
1295 size_t max_ent = find_max_entity_count(part_count, local_mesh, global, blocks, nodesets,
1296 sidesets, edgeblocks, faceblocks);
1297 std::vector<T> values(max_ent);
1298
1299 size_t max_global_ent = find_max_global_entity_count(
1300 global, glob_blocks, glob_nsets, glob_ssets, glob_edgeblocks, glob_faceblocks);
1301 std::vector<T> master_values(max_global_ent);
1302
1303 // Stage II. Extracting transient variable data.
1304 // loop over time steps
1305
1306 if (num_time_steps == 0 && element_vars.add_processor_id()) {
1307 // Add a fake timestep with just the processor id information.
1308 // If adding the processor_id field, do it here...
1309 T time_val = 0.0;
1310 error = ex_put_time(ExodusFile::output(), 1, &time_val);
1311 if (error < 0) {
1312 exodus_error(__LINE__);
1313 }
1314
1315 std::vector<T> proc;
1316 add_processor_variable(ExodusFile::output(), part_count, start_part, global, blocks,
1317 glob_blocks, local_element_to_global, 1,
1318 element_vars.index_[element_vars.count()], proc);
1319 }
1320
1321 // Determine if user wants a subset of timesteps transferred to the output file.
1322 int ts_min = interFace.step_min();
1323 int ts_max = interFace.step_max();
1324 int ts_step = interFace.step_interval();
1325
1326 if (ts_min < 0) {
1327 ts_min = num_time_steps + 1 + ts_min;
1328 }
1329 if (ts_max < 0) {
1330 ts_max = num_time_steps + 1 + ts_max;
1331 }
1332
1333 // Time steps for output file
1334 int time_step_out = 0;
1335 T sentinel = static_cast<T>(-FLT_MAX);
1336 T min_time_to_write = sentinel;
1337
1338 if (interFace.append()) {
1339 // See how many steps already exist on the output database
1340 // and the corresponding time.
1341 int nstep = ex_inquire_int(ExodusFile::output(), EX_INQ_TIME);
1342
1343 // Get the time corresponding to this step...
1344 error = ex_get_time(ExodusFile::output(), nstep, &min_time_to_write);
1345 if (error < 0) {
1346 exodus_error(__LINE__);
1347 }
1348
1349 time_step_out = nstep;
1350 }
1351
1352 ts_max = std::min(ts_max, num_time_steps);
1353 if (ts_min <= ts_max) {
1354 if (debug_level & 1) {
1355 fmt::print("{}", time_stamp(tsFormat));
1356 }
1357 if (rank == 0) {
1358 fmt::print("\tTransferring step {} to step {} by {}\n", ts_min, ts_max, ts_step);
1359 }
1360 }
1361
1362 // Determine how many steps will be written...
1363 int output_steps = (ts_max - ts_min) / ts_step + 1;
1364 int subcycles = interFace.subcycle();
1365
1366 double start_time = seacas_timer();
1367
1368 for (int time_step = ts_min - 1; time_step < ts_max; time_step += ts_step) {
1369 time_step_out++;
1370
1371 T time_val = -std::numeric_limits<T>::max();
1372 {
1373 // read in and write out the time step information
1374 ExodusFile id(0);
1375
1376 error = ex_get_time(id, time_step + 1, &time_val);
1377 if (error < 0) {
1378 exodus_error(__LINE__);
1379 }
1380 if (time_val <= min_time_to_write) {
1381 continue;
1382 }
1383
1384 if (min_time_to_write != sentinel) {
1385 if (rank == 0) {
1386 fmt::print("\tAppend Mode: Skipping {} input steps to align times with already written "
1387 "steps on output file.\n\n",
1388 time_step - (ts_min - 1));
1389 }
1390 min_time_to_write = sentinel;
1391 }
1392
1393 error = ex_put_time(ExodusFile::output(), time_step_out, &time_val);
1394 if (error < 0) {
1395 exodus_error(__LINE__);
1396 }
1397
1398 for (int p = 1; p < part_count; p++) {
1399 ExodusFile idp(p);
1400 T proc_time_val = 0.0;
1401 error = ex_get_time(idp, time_step + 1, &proc_time_val);
1402 if (error < 0) {
1403 exodus_error(__LINE__);
1404 }
1405 if (proc_time_val != time_val) {
1406 fmt::print(
1407 stderr,
1408 "WARNING: (EPU) At step {}, the times on processors {} and {} do not match:\n"
1409 " {:.8} vs {:.8} (absolute diff: {:.8})\n"
1410 " This may indicate a corrupt database.\n",
1411 time_step + 1, start_part, p + start_part, time_val, proc_time_val,
1412 std::abs(time_val - proc_time_val));
1413 }
1414 }
1415
1416 // NOTE: Assuming that each processor has the exact same global
1417 // information
1418 if (global_vars.count(InOut::OUT) > 0) {
1419 if (debug_level & 1) {
1420 if (rank == 0) {
1421 fmt::print("{}Global Variables...\n", time_stamp(tsFormat));
1422 }
1423 }
1424 error = ex_get_var(id, time_step + 1, EX_GLOBAL, 0, 0, global_vars.count(),
1425 Data(global_values));
1426 if (error < 0) {
1427 exodus_error(__LINE__);
1428 }
1429 // Map ...
1430 for (int ig = 0; ig < global_vars.count(InOut::IN); ig++) {
1431 if (global_vars.index_[ig] > 0) {
1432 SMART_ASSERT(ig < (int)global_values.size());
1433 output_global_values[global_vars.index_[ig] - 1] = global_values[ig];
1434 }
1435 }
1436 error = ex_put_var(ExodusFile::output(), time_step_out, EX_GLOBAL, 1, 0,
1437 global_vars.count(InOut::OUT), Data(output_global_values));
1438 if (error < 0) {
1439 exodus_error(__LINE__);
1440 }
1441
1442 // Check global variable consistency...
1443 if (debug_level & 512) {
1444 std::vector<T> proc_global_values(global_vars.count(InOut::IN));
1445 for (int p = 1; p < part_count; p++) {
1446 ExodusFile idp(p);
1447 error = ex_get_var(idp, time_step + 1, EX_GLOBAL, 0, 0, global_vars.count(InOut::IN),
1448 Data(proc_global_values));
1449 if (error < 0) {
1450 exodus_error(__LINE__);
1451 }
1452 for (int ig = 0; ig < global_vars.count(InOut::IN); ig++) {
1453 if (proc_global_values[ig] != global_values[ig]) {
1454 fmt::print(
1455 stderr,
1456 fmt::runtime(
1457 "At step {:{}}, Global Variable {:{}}, P{:0{}} = {:15.8g}, P{:0{}} = "
1458 "{:15.8g}\n"),
1459 time_step + 1, ts_max + 1, ig + 1, get_width(global_vars.count(InOut::IN)),
1460 start_part, get_width(interFace.processor_count()), start_part + p,
1461 get_width(interFace.processor_count()), proc_global_values[ig]);
1462 }
1463 }
1464 }
1465 }
1466 }
1467 }
1468
1469 // Original: Total Execution Time = 20.55 seconds, Maximum memory = 3,826 MiBytes.
1470 // Refactor: Total Execution Time = 18.72 seconds, Maximum memory = 2,252 MiBytes. (nodal
1471 // variables only)
1472 // Total Execution Time = 18.25 seconds, Maximum memory = 869 MiBytes. (all
1473 // variables) Total Execution Time = 18.07 seconds, Maximum memory = 863 MiBytes.
1474 // (all variables, share master_value array)
1475 // ========================================================================
1476 // Nodal Values...
1477 if (debug_level & 1) {
1478 if (rank == 0) {
1479 fmt::print("{}Nodal Variables...\n", time_stamp(tsFormat));
1480 }
1481 }
1482
1483 if (nodal_vars.count(InOut::OUT) > 0) {
1484 T fill_val = -std::numeric_limits<T>::max();
1485 for (int i = 0; i < nodal_vars.count(InOut::IN); i++) {
1486 if (debug_level & 2) {
1487 std::fill(master_values.begin(), master_values.end(), fill_val);
1488 }
1489
1490 for (int p = 0; p < part_count; p++) {
1491 ExodusFile id(p);
1492
1493 size_t node_count = local_mesh[p].nodeCount;
1494 if (nodal_vars.index_[i] > 0) {
1495 error = ex_get_var(id, time_step + 1, EX_NODAL, i + 1, 0, node_count, Data(values));
1496 if (error < 0) {
1497 exodus_error(__LINE__);
1498 }
1499
1500 if (debug_level & 2) {
1501 for (size_t j = 0; j < node_count; j++) {
1502 size_t nodal_value = local_node_to_global[p][j];
1503 if (master_values[nodal_value] != fill_val &&
1504 master_values[nodal_value] != values[j]) {
1505 fmt::print(stderr, "Variable {}, Node {}, old = {}, new = {}\n", i + 1,
1506 fmt::group_digits(nodal_value), master_values[nodal_value],
1507 values[j]);
1508 }
1509 }
1510 }
1511
1512 if (interFace.sum_shared_nodes()) {
1513 // sum values into master nodal value information. Note
1514 // that for non-shared nodes, this will be the same as a
1515 // copy; for shared nodes, it will be a true sum.
1516 for (size_t j = 0; j < node_count; j++) {
1517 // Map local nodal value to global location...
1518 size_t nodal_value = local_node_to_global[p][j];
1519 master_values[nodal_value] += values[j];
1520 }
1521 }
1522 else {
1523 // copy values to master nodal value information
1524 for (size_t j = 0; j < node_count; j++) {
1525 // Map local nodal value to global location...
1526 size_t nodal_value = local_node_to_global[p][j];
1527 master_values[nodal_value] = values[j];
1528 }
1529 }
1530 }
1531 }
1532 // output nodal variable info. for specified time step
1533 int i_out = nodal_vars.index_[i];
1534 SMART_ASSERT(i_out <= nodal_vars.count(InOut::OUT));
1535 error = ex_put_var(ExodusFile::output(), time_step_out, EX_NODAL, i_out, 0,
1536 global.nodeCount, Data(master_values));
1537 if (error < 0) {
1538 exodus_error(__LINE__);
1539 }
1540 }
1541 }
1542
1543 // ========================================================================
1544 // Extracting element transient variable data
1545 if (debug_level & 1) {
1546 if (rank == 0) {
1547 fmt::print("{}Element Variables...\n", time_stamp(tsFormat));
1548 }
1549 }
1550
1551 if (element_vars.count(InOut::IN) > 0) {
1552 read_write_master_values(element_vars, global, glob_blocks, local_mesh, blocks,
1553 master_values, values, part_count, time_step, time_step_out,
1554 local_element_to_global);
1555 }
1556
1557 // If adding the processor_id field, do it here...
1558 // Use the output time step for writing data
1559 if (element_vars.add_processor_id()) {
1560 std::vector<T> proc;
1561 add_processor_variable(ExodusFile::output(), part_count, start_part, global, blocks,
1562 glob_blocks, local_element_to_global, time_step_out,
1563 element_vars.index_[element_vars.count(InOut::IN)], proc);
1564 }
1565
1566 // ========================================================================
1567 // Extracting sideset transient variable data
1568 if (!interFace.omit_sidesets()) {
1569 if (debug_level & 1) {
1570 if (rank == 0) {
1571 fmt::print("{}Sideset Variables...\n", time_stamp(tsFormat));
1572 }
1573 }
1574
1575 if (sideset_vars.count(InOut::IN) > 0) {
1576 read_write_master_values(sideset_vars, global, glob_ssets, local_mesh, sidesets,
1577 master_values, values, part_count, time_step, time_step_out,
1578 local_element_to_global);
1579 }
1580 }
1581
1582 if (!interFace.omit_nodesets()) {
1583 // ========================================================================
1584 // Extracting nodeset transient variable data
1585 if (debug_level & 1) {
1586 if (rank == 0) {
1587 fmt::print("{}Nodeset Variables...\n", time_stamp(tsFormat));
1588 }
1589 }
1590
1591 if (nodeset_vars.count(InOut::IN) > 0) {
1592 read_write_master_values(nodeset_vars, global, glob_nsets, local_mesh, nodesets,
1593 master_values, values, part_count, time_step, time_step_out,
1594 local_element_to_global);
1595 }
1596 }
1597
1598 if (!interFace.omit_edgeblocks()) {
1599 // ========================================================================
1600 // Extracting edgeblock transient variable data
1601 if (debug_level & 1) {
1602 if (rank == 0) {
1603 fmt::print("{}Edgeblock Variables...\n", time_stamp(tsFormat));
1604 }
1605 }
1606
1607 if (edgeblock_vars.count(InOut::IN) > 0) {
1608 read_write_master_values(edgeblock_vars, global, glob_edgeblocks, local_mesh, edgeblocks,
1609 master_values, values, part_count, time_step, time_step_out,
1610 local_edge_to_global);
1611 }
1612 }
1613
1614 if (!interFace.omit_faceblocks()) {
1615 // ========================================================================
1616 // Extracting faceblock transient variable data
1617 if (debug_level & 1) {
1618 if (rank == 0) {
1619 fmt::print("{}Faceblock Variables...\n", time_stamp(tsFormat));
1620 }
1621 }
1622
1623 if (faceblock_vars.count(InOut::IN) > 0) {
1624 read_write_master_values(faceblock_vars, global, glob_faceblocks, local_mesh, faceblocks,
1625 master_values, values, part_count, time_step, time_step_out,
1626 local_face_to_global);
1627 }
1628 }
1629
1630 // ========================================================================
1631 if (debug_level & 1) {
1632 fmt::print("{}", time_stamp(tsFormat));
1633 }
1634
1635 if (subcycles > 2) {
1636 if (rank == 0) {
1637 fmt::print("{}/{} ", cycle + 1, subcycles);
1638 }
1639 }
1640
1641 double cur_time = seacas_timer();
1642 double elapsed = cur_time - start_time;
1643 double time_per_step = elapsed / time_step_out;
1644 double percentage_done = (time_step_out * 100.0) / output_steps;
1645 double estimated_remaining = time_per_step * (output_steps - time_step_out);
1646 fmt::print("Wrote step {:6}, time {:8.4e}\t\t[{:5.1f}%, Elapsed={}, ETA={}] \r",
1647 fmt::group_digits(time_step + 1), time_val, percentage_done, format_time(elapsed),
1648 format_time(estimated_remaining));
1649 if (debug_level & 1) {
1650 fmt::print("\n");
1651 }
1652 }
1653 /*************************************************************************/
1654 // FINALIZE program
1655 if (debug_level & 1) {
1656 fmt::print("{}", time_stamp(tsFormat));
1657 }
1658 if (subcycles > 2) {
1659 fmt::print("{}/{} ", cycle + 1, subcycles);
1660 }
1661 }
1662 fmt::print("\n\nTotal Execution Time = {:.2f} seconds, Maximum memory = {} MiBytes.\n******* "
1663 "END *******\n",
1664 seacas_timer() - execution_time,
1665 fmt::group_digits((get_hwm_memory_info() + 1024 * 1024 - 1) / (1024 * 1024)));
1666 return 0;
1667 }
1668
1669 namespace {
1670 void get_put_assemblies(int in, int out, Excn::Mesh &global)
1671 {
1672 if (global.assemblyCount > 0) {
1673 std::vector<ex_assembly> assemblies(global.assemblyCount);
1674 for (int i = 0; i < global.assemblyCount; i++) {
1675 assemblies[i].name = nullptr;
1676 assemblies[i].entity_list = nullptr;
1677 }
1678 ex_get_assemblies(in, Data(assemblies));
1679
1680 for (int i = 0; i < global.assemblyCount; i++) {
1681 assemblies[i].entity_list = new int64_t[assemblies[i].entity_count];
1682 }
1683
1684 // Now get the assembly entity lists...
1685 ex_get_assemblies(in, Data(assemblies));
1686 ex_put_assemblies(out, assemblies.size(), Data(assemblies));
1687
1688 for (int i = 0; i < global.assemblyCount; i++) {
1689 delete[] assemblies[i].entity_list;
1690 // This is `calloc`d in `ex_get_assemblies` call.
1691 free(assemblies[i].name);
1692 }
1693 }
1694 }
1695
1696 template <typename T> void get_put_coordinate_frames(int id, int id_out)
1697 {
1698 int num_frames = ex_inquire_int(id, EX_INQ_COORD_FRAMES);
1699 if (num_frames <= 0) {
1700 return;
1701 }
1702
1703 ExodusIdVector ids(num_frames);
1704 std::vector<T> coordinates(9 * num_frames);
1705 std::vector<char> tags(num_frames);
1706
1707 int error = ex_get_coordinate_frames(id, &num_frames, Data(ids), Data(coordinates), Data(tags));
1708 if (error < 0) {
1709 exodus_error(__LINE__);
1710 }
1711
1712 // Now output to the combined file...
1713 error = ex_put_coordinate_frames(id_out, num_frames, Data(ids), Data(coordinates), Data(tags));
1714 if (error < 0) {
1715 exodus_error(__LINE__);
1716 }
1717 }
1718
1719 void get_put_qa(int id, int id_out)
1720 {
1721 // NOTE: Assuming info and QA records for all processors
1722 int error = 0;
1723
1724 // I. Get and store info strings, if they exist
1725 int num_info_records = ex_inquire_int(id, EX_INQ_INFO);
1726 auto *info_records = new char *[num_info_records + 1];
1727 int info_string_len = MAX_LINE_LENGTH;
1728
1729 {
1730 for (int i = 0; i < num_info_records + 1; i++) {
1731 info_records[i] = new char[info_string_len + 1];
1732 memset(info_records[i], '\0', info_string_len + 1);
1733 }
1734 }
1735
1736 if (num_info_records > 0) {
1737 error = ex_get_info(id, info_records);
1738 if (error < 0) {
1739 exodus_error(__LINE__);
1740 }
1741 }
1742
1743 // Add an info record for EPU
1744 add_info_record(info_records[num_info_records], MAX_LINE_LENGTH);
1745
1746 error = ex_put_info(id_out, num_info_records + 1, info_records);
1747 if (error < 0) {
1748 exodus_error(__LINE__);
1749 }
1750
1751 {
1752 for (int i = 0; i < num_info_records + 1; i++) {
1753 delete[] info_records[i];
1754 }
1755 delete[] info_records;
1756 }
1757
1758 // II. Get and store QA records, if they exist
1759 struct qa_element
1760 {
1761 char *qa_record[1][4];
1762 };
1763
1764 int num_qa_records = ex_inquire_int(id, EX_INQ_QA);
1765 std::vector<qa_element> qaRecord(num_qa_records + 1);
1766 for (int i = 0; i < num_qa_records + 1; i++) {
1767 for (int j = 0; j < 4; j++) {
1768 qaRecord[i].qa_record[0][j] = new char[MAX_STR_LENGTH + 1];
1769 qaRecord[i].qa_record[0][j][0] = '\0';
1770 }
1771 }
1772 if (num_qa_records != 0) {
1773 error = ex_get_qa(id, qaRecord[0].qa_record);
1774 if (error < 0) {
1775 exodus_error(__LINE__);
1776 }
1777 }
1778
1779 std::string buffer;
1780
1781 copy_string(qaRecord[num_qa_records].qa_record[0][0], qainfo[0], MAX_STR_LENGTH + 1); // Code
1782 copy_string(qaRecord[num_qa_records].qa_record[0][1], qainfo[2], MAX_STR_LENGTH + 1); // Version
1783
1784 time_t date_time = std::time(nullptr);
1785 #if defined __NVCC__
1786 auto *lt = std::localtime(&date_time);
1787 buffer = fmt::format("{:%Y/%m/%d}", *lt);
1788 #else
1789 auto const lt = fmt::localtime(date_time);
1790 buffer = fmt::format("{:%Y/%m/%d}", lt);
1791 #endif
1792 copy_string(qaRecord[num_qa_records].qa_record[0][2], buffer, MAX_STR_LENGTH + 1);
1793
1794 #if defined __NVCC__
1795 buffer = fmt::format("{:%H:%M:%S}", *lt);
1796 #else
1797 buffer = fmt::format("{:%H:%M:%S}", lt);
1798 #endif
1799 copy_string(qaRecord[num_qa_records].qa_record[0][3], buffer, MAX_STR_LENGTH + 1);
1800
1801 error = ex_put_qa(id_out, num_qa_records + 1, qaRecord[0].qa_record);
1802 if (error < 0) {
1803 exodus_error(__LINE__);
1804 }
1805
1806 for (int i = 0; i < num_qa_records + 1; i++) {
1807 for (int j = 0; j < 4; j++) {
1808 delete[] qaRecord[i].qa_record[0][j];
1809 }
1810 }
1811 }
1812
1813 template <typename T, typename INT>
1814 void get_put_coordinates(Mesh &global, int part_count, std::vector<Mesh> &local_mesh,
1815 const std::vector<std::vector<INT>> &local_node_to_global)
1816 {
1817 T FillValue = static_cast<T>(FILL_VALUE);
1818 SMART_ASSERT(sizeof(T) == ExodusFile::io_word_size());
1819 std::vector<T> x(global.nodeCount);
1820 std::vector<T> y(global.nodeCount);
1821 std::vector<T> z(global.nodeCount);
1822
1823 if (debug_level & 8) {
1824 std::fill(x.begin(), x.end(), FillValue);
1825 std::fill(y.begin(), y.end(), FillValue);
1826 std::fill(z.begin(), z.end(), FillValue);
1827 }
1828
1829 int error = 0;
1830 for (int p = 0; p < part_count; p++) {
1831 get_coordinates(ExodusFile(p), global.dimensionality, local_mesh[p].nodeCount,
1832 local_node_to_global, p, x, y, z);
1833 } // end for p=0..part_count
1834
1835 // Get Coordinate Names
1836 // NOTE: Assuming coordinate names should be
1837 // the same for all files/processors therefore, only one
1838 // file/processor needs to be loaded
1839 get_put_coordinate_names(ExodusFile(0), ExodusFile::output(), global.dimensionality);
1840
1841 // Write out coordinate information
1842 error = ex_put_coord(ExodusFile::output(), Data(x), Data(y), Data(z));
1843 if (error < 0) {
1844 exodus_error(__LINE__);
1845 }
1846 }
1847
1848 void get_put_coordinate_names(int in, int out, int dimensionality)
1849 {
1850 int error = 0;
1851 char **coordinate_names = get_name_array(dimensionality, ExodusFile::max_name_length());
1852
1853 error = ex_get_coord_names(in, coordinate_names);
1854 if (error < 0) {
1855 exodus_error(__LINE__);
1856 }
1857 error = ex_put_coord_names(out, coordinate_names);
1858 if (error < 0) {
1859 exodus_error(__LINE__);
1860 }
1861 LOG("Wrote coordinate names...\n");
1862
1863 free_name_array(coordinate_names, dimensionality);
1864 }
1865
1866 template <typename INT>
1867 void get_put_nodal_communication_map(int part_count,
1868 const std::vector<std::vector<INT>> &local_node_to_global,
1869 const std::vector<int> &processor_map, int output_processor)
1870 {
1871 int error = 0;
1872 std::vector<std::pair<INT, int>> node_cmap;
1873 for (int p = 0; p < part_count; p++) {
1874 INT num_int_nodes, num_bor_nodes, num_ext_nodes, num_int_elems, num_bor_elems;
1875 INT num_node_cmaps, num_elem_cmaps;
1876 ex_get_loadbal_param(ExodusFile(p), &num_int_nodes, &num_bor_nodes, &num_ext_nodes,
1877 &num_int_elems, &num_bor_elems, &num_node_cmaps, &num_elem_cmaps, p);
1878
1879 std::vector<INT> nodeCmapIds(num_node_cmaps);
1880 std::vector<INT> nodeCmapNodeCnts(num_node_cmaps);
1881 ex_get_cmap_params(ExodusFile(p), Data(nodeCmapIds), Data(nodeCmapNodeCnts), nullptr, nullptr,
1882 p);
1883
1884 int64_t my_node_count =
1885 std::accumulate(nodeCmapNodeCnts.begin(), nodeCmapNodeCnts.end(), int64_t(0));
1886 std::vector<INT> nodes(my_node_count);
1887 std::vector<INT> procs(my_node_count);
1888
1889 int64_t cm_offset = 0;
1890 for (INT i = 0; i < num_node_cmaps; i++) {
1891 ex_get_node_cmap(ExodusFile(p), nodeCmapIds[i], &nodes[cm_offset], &procs[cm_offset], p);
1892 cm_offset += nodeCmapNodeCnts[i];
1893 }
1894 for (size_t i = 0; i < nodes.size(); i++) {
1895 auto output_proc = processor_map[procs[i]];
1896 if (output_proc != output_processor) {
1897 node_cmap.emplace_back(local_node_to_global[p][nodes[i] - 1] + 1,
1898 processor_map[procs[i]]);
1899 }
1900 }
1901 } // end for p=0..part_count
1902 uniquify(node_cmap);
1903
1904 std::vector<INT> gnodes;
1905 std::vector<INT> gprocs;
1906 gnodes.reserve(node_cmap.size());
1907 gprocs.reserve(node_cmap.size());
1908
1909 for (auto &[node, proc] : node_cmap) {
1910 gnodes.push_back(node);
1911 gprocs.push_back(proc);
1912 }
1913
1914 // NOTE: This is inefficient in general since going in and out of define mode.
1915 // Would be better to do this at file-creation time, but this encapsulates all
1916 // code related to node communication map output to this routine...
1917
1918 // Write out nodal communication map information
1919 error = ex_put_init_info(ExodusFile::output(), processor_map.back() + 1, 1, (char *)"P");
1920 if (error < 0) {
1921 exodus_error(__LINE__);
1922 }
1923
1924 error = ex_put_loadbal_param(ExodusFile::output(), 0, 0, 0, 0, 0, 1, 0, output_processor);
1925 if (error < 0) {
1926 exodus_error(__LINE__);
1927 }
1928
1929 std::array<INT, 1> ids{1};
1930 std::array<INT, 1> cnts{(INT)gnodes.size()};
1931 error = ex_put_cmap_params(ExodusFile::output(), Data(ids), Data(cnts), nullptr, nullptr,
1932 output_processor);
1933 if (error < 0) {
1934 exodus_error(__LINE__);
1935 }
1936
1937 error = ex_put_node_cmap(ExodusFile::output(), 1, Data(gnodes), Data(gprocs), output_processor);
1938 if (error < 0) {
1939 exodus_error(__LINE__);
1940 }
1941 }
1942
1943 template <typename T, typename INT>
1944 void get_coordinates(int id, int dimensionality, size_t num_nodes,
1945 const std::vector<std::vector<INT>> &local_node_to_global, int proc,
1946 std::vector<T> &x, std::vector<T> &y, std::vector<T> &z)
1947 {
1948 SMART_ASSERT(sizeof(T) == ExodusFile::io_word_size());
1949 T FillValue = static_cast<T>(FILL_VALUE);
1950 int error = 0;
1951 std::vector<T> local_x(num_nodes);
1952 std::vector<T> local_y(num_nodes);
1953 std::vector<T> local_z(num_nodes);
1954
1955 error = ex_get_coord(id, Data(local_x), Data(local_y), Data(local_z));
1956 if (error < 0) {
1957 exodus_error(__LINE__);
1958 }
1959
1960 // Check for 2D or 3D coordinates
1961
1962 if (dimensionality == 3) {
1963 if (debug_level & 8) {
1964 for (size_t i = 0; i < num_nodes; i++) {
1965 size_t node = local_node_to_global[proc][i];
1966 if (x[node] != FillValue && y[node] != FillValue && z[node] != FillValue) {
1967 if (x[node] != local_x[i] || y[node] != local_y[i] || z[node] != local_z[i]) {
1968 fmt::print(stderr,
1969 "\nWARNING: Node {} has different coordinates in at least two files.\n"
1970 " cur value = {:14.6e} {:14.6e} {:14.6e}\n"
1971 " new value = {:14.6e} {:14.6e} {:14.6e} from processor {}\n",
1972 fmt::group_digits(node + 1), x[node], y[node], z[node], local_x[i],
1973 local_y[i], local_z[i], proc);
1974 }
1975 }
1976 }
1977 }
1978
1979 for (size_t i = 0; i < num_nodes; i++) {
1980
1981 // The following WILL overwrite x[node],y[node],z[node] if node is the
1982 // same for different processors
1983 size_t node = local_node_to_global[proc][i];
1984 x[node] = local_x[i];
1985 y[node] = local_y[i];
1986 z[node] = local_z[i];
1987 }
1988 }
1989 else if (dimensionality == 2) {
1990 if (debug_level & 8) {
1991 for (size_t i = 0; i < num_nodes; i++) {
1992 size_t node = local_node_to_global[proc][i];
1993 if (x[node] != FillValue && y[node] != FillValue) {
1994 if (x[node] != local_x[i] || y[node] != local_y[i]) {
1995 fmt::print(stderr,
1996 "\nWARNING: Node {} has different coordinates in at least two files.\n"
1997 " cur value = {:14.6e} {:14.6e}\n"
1998 " new value = {:14.6e} {:14.6e} from processor {}\n",
1999 fmt::group_digits(node + 1), x[node], y[node], local_x[i], local_y[i],
2000 proc);
2001 }
2002 }
2003 }
2004 }
2005 for (size_t i = 0; i < num_nodes; i++) {
2006
2007 // The following WILL overwrite x[node],y[node] if node is the same for
2008 // different processors
2009 size_t node = local_node_to_global[proc][i];
2010
2011 x[node] = local_x[i];
2012 y[node] = local_y[i];
2013 }
2014 }
2015 else { // dimensionality == 1
2016 if (debug_level & 8) {
2017 for (size_t i = 0; i < num_nodes; i++) {
2018 size_t node = local_node_to_global[proc][i];
2019 if (x[node] != FillValue && y[node] != FillValue) {
2020 if (x[node] != local_x[i]) {
2021 fmt::print(stderr,
2022 "\nWARNING: Node {} has different coordinates in at least two files.\n"
2023 " cur value = {:14.6e}\tnew value = {:14.6e} from processor {}\n",
2024 fmt::group_digits(node + 1), x[node], local_x[i], proc);
2025 }
2026 }
2027 }
2028 }
2029 for (size_t i = 0; i < num_nodes; i++) {
2030
2031 // The following WILL overwrite x[node] if node is the same for
2032 // different processors
2033 size_t node = local_node_to_global[proc][i];
2034 x[node] = local_x[i];
2035 }
2036 }
2037 }
2038
2039 void get_element_blocks(int part_count, const std::vector<Mesh> &local_mesh, const Mesh &global,
2040 std::vector<std::vector<Block>> &blocks, std::vector<Block> &glob_blocks)
2041 {
2042 LOG("\n\n**** GET BLOCK INFORMATION (INCL. ELEMENT ATTRIBUTES) ****\n");
2043
2044 for (int ip = 0; ip < part_count; ip++) {
2045 blocks[ip].resize(local_mesh[ip].count(Excn::ObjectType::EBLK));
2046 }
2047
2048 if (rank == 0) {
2049 fmt::print("Global block count = {}\n", global.count(Excn::ObjectType::EBLK));
2050 }
2051
2052 ExodusIdVector block_id(global.count(Excn::ObjectType::EBLK));
2053
2054 int error = 0;
2055 for (int p = 0; p < part_count; p++) {
2056 ExodusFile id(p);
2057
2058 error = ex_get_ids(id, EX_ELEM_BLOCK, Data(block_id));
2059 if (error < 0) {
2060 exodus_error(__LINE__);
2061 }
2062
2063 // Check that the block id ordering is consistent among files...
2064 if (p > 0) {
2065 for (size_t b = 0; b < global.count(Excn::ObjectType::EBLK); b++) {
2066 if (blocks[0][b].id != block_id[b]) {
2067 std::ostringstream errmsg;
2068 fmt::print(errmsg,
2069 "ERROR: (EPU) The internal element block id ordering for part {}\n"
2070 " is not consistent with the ordering for part 0.\n",
2071 p);
2072 throw std::runtime_error(errmsg.str());
2073 }
2074 }
2075 }
2076
2077 if ((debug_level & 4) != 0U) {
2078 fmt::print("\nGetting element block info for processor {}...\n", p);
2079 }
2080 else {
2081 if (p == 0) {
2082 LOG("\nGetting element block info.\n");
2083 }
2084 }
2085
2086 for (size_t b = 0; b < global.count(Excn::ObjectType::EBLK); b++) {
2087 if ((debug_level & 4) != 0U) {
2088 fmt::print("Block {}, Id = {}", b, block_id[b]);
2089 }
2090
2091 ex_block temp_block{};
2092 temp_block.id = block_id[b];
2093 temp_block.type = EX_ELEM_BLOCK;
2094 error = ex_get_block_param(id, &temp_block);
2095 if (error < 0) {
2096 exodus_error(__LINE__);
2097 }
2098
2099 std::vector<char> name(Excn::ExodusFile::max_name_length() + 1);
2100 error = ex_get_name(id, EX_ELEM_BLOCK, block_id[b], Data(name));
2101 if (error < 0) {
2102 exodus_error(__LINE__);
2103 }
2104
2105 blocks[p][b].id = block_id[b];
2106 if (name[0] != '\0') {
2107 blocks[p][b].name_ = Data(name);
2108 }
2109 if (p == 0) {
2110 glob_blocks[b].id = block_id[b];
2111 if (name[0] != '\0') {
2112 glob_blocks[b].name_ = Data(name);
2113 }
2114 }
2115
2116 if (temp_block.num_entry != 0) {
2117 blocks[p][b].elementCount = temp_block.num_entry;
2118 blocks[p][b].nodesPerElement = temp_block.num_nodes_per_entry;
2119 blocks[p][b].attributeCount = temp_block.num_attribute;
2120 blocks[p][b].offset_ = temp_block.num_entry;
2121 blocks[p][b].position_ = b;
2122 copy_string(blocks[p][b].elType, temp_block.topology);
2123
2124 glob_blocks[b].elementCount += temp_block.num_entry;
2125 glob_blocks[b].nodesPerElement = temp_block.num_nodes_per_entry;
2126 glob_blocks[b].attributeCount = temp_block.num_attribute;
2127 glob_blocks[b].position_ = (int)b;
2128 copy_string(glob_blocks[b].elType, temp_block.topology);
2129 }
2130
2131 if (temp_block.num_attribute > 0 && glob_blocks[b].attributeNames.empty()) {
2132 // Get attribute names. Assume the same on all processors
2133 // on which the block exists.
2134 char **names = get_name_array(temp_block.num_attribute, ExodusFile::max_name_length());
2135
2136 error = ex_get_attr_names(id, EX_ELEM_BLOCK, block_id[b], names);
2137 if (error < 0) {
2138 exodus_error(__LINE__);
2139 }
2140 for (int i = 0; i < temp_block.num_attribute; i++) {
2141 glob_blocks[b].attributeNames.emplace_back(names[i]);
2142 }
2143 free_name_array(names, temp_block.num_attribute);
2144 }
2145 if ((debug_level & 4) != 0U) {
2146 fmt::print(", Name = '{}', Elements = {:12}, Nodes/element = {}, Attributes = {}\n",
2147 blocks[p][b].name_, fmt::group_digits(blocks[p][b].entity_count()),
2148 blocks[p][b].nodesPerElement, blocks[p][b].attributeCount);
2149 }
2150 }
2151 } // end for p=0..part_count
2152
2153 // Convert block_offset from elements/block/processor to true offset
2154 for (int p = 0; p < part_count; p++) {
2155 size_t sum = 0;
2156 for (size_t b = 0; b < global.count(Excn::ObjectType::EBLK); b++) {
2157 size_t save = blocks[p][b].offset_;
2158 blocks[p][b].offset_ = sum;
2159 sum += save;
2160 }
2161 }
2162 }
2163
2164 template <typename T, typename INT>
2165 void put_element_blocks(int part_count, int start_part, std::vector<std::vector<Block>> &blocks,
2166 std::vector<Block> &glob_blocks,
2167 const std::vector<std::vector<INT>> &local_node_to_global,
2168 const std::vector<std::vector<INT>> &local_element_to_global)
2169 {
2170 SMART_ASSERT(sizeof(T) == ExodusFile::io_word_size());
2171 int global_num_blocks = glob_blocks.size();
2172
2173 LOG("\nReading and Writing element connectivity & attributes\n");
2174
2175 std::vector<INT> linkage;
2176 std::vector<T> attributes;
2177 std::vector<INT> local_linkage;
2178 std::vector<T> local_attr;
2179
2180 for (int b = 0; b < global_num_blocks; b++) {
2181
2182 if (debug_level & 4) {
2183 fmt::print("\nOutput element block info for...\n"
2184 "Block {}, Id = {}, Name = '{}', Elements = {:12}, Nodes/element = {}, "
2185 "Attributes = {}\n"
2186 "B{}:\t",
2187 b, glob_blocks[b].id, glob_blocks[b].name_,
2188 fmt::group_digits(glob_blocks[b].entity_count()), glob_blocks[b].nodesPerElement,
2189 glob_blocks[b].attributeCount, b);
2190 }
2191
2192 size_t max_nodes = glob_blocks[b].entity_count() * glob_blocks[b].nodesPerElement;
2193 linkage.resize(max_nodes);
2194 attributes.resize(static_cast<size_t>(glob_blocks[b].attributeCount) *
2195 glob_blocks[b].entity_count());
2196
2197 int error = 0;
2198 for (int p = 0; p < part_count; p++) {
2199 ExodusFile id(p);
2200
2201 size_t global_pos;
2202 size_t global_block_pos;
2203
2204 if (blocks[p][b].entity_count() > 0) { // non-zero length block
2205
2206 if (debug_level & 4) {
2207 fmt::print("#");
2208 }
2209 size_t maximum_nodes = blocks[p][b].entity_count();
2210 maximum_nodes *= blocks[p][b].nodesPerElement;
2211 local_linkage.resize(maximum_nodes);
2212
2213 ex_entity_id bid = blocks[p][b].id;
2214 error = ex_get_conn(id, EX_ELEM_BLOCK, bid, Data(local_linkage), nullptr, nullptr);
2215 if (error < 0) {
2216 fmt::print(
2217 stderr,
2218 "ERROR: (EPU) Cannot get element block connectivity for block {} on part {}.\n",
2219 bid, p + start_part);
2220 exodus_error(__LINE__);
2221 }
2222 size_t pos = 0;
2223 size_t goffset = glob_blocks[b].offset_;
2224 size_t element_count = blocks[p][b].entity_count();
2225 size_t boffset = blocks[p][b].offset_;
2226 size_t npe = blocks[p][b].nodesPerElement;
2227 const std::vector<INT> &proc_loc_elem_to_global = local_element_to_global[p];
2228 const std::vector<INT> &proc_loc_node_to_global = local_node_to_global[p];
2229
2230 for (size_t e = 0; e < element_count; e++) {
2231 global_block_pos = proc_loc_elem_to_global[(e + boffset)] - goffset;
2232 global_pos = global_block_pos * npe;
2233
2234 for (size_t n = 0; n < npe; n++) {
2235 size_t node = proc_loc_node_to_global[local_linkage[pos++] - 1];
2236 linkage[global_pos++] = node + 1;
2237 }
2238 }
2239
2240 // Get attributes list, if it exists
2241 if (blocks[p][b].attributeCount > 0) {
2242 size_t max_attr = blocks[p][b].entity_count() * blocks[p][b].attributeCount;
2243 local_attr.resize(max_attr);
2244
2245 error = ex_get_attr(id, EX_ELEM_BLOCK, blocks[p][b].id, Data(local_attr));
2246 if (error < 0) {
2247 exodus_error(__LINE__);
2248 }
2249
2250 pos = 0;
2251
2252 size_t att_count = blocks[p][b].attributeCount;
2253 for (size_t e = 0; e < element_count; e++) {
2254 // global_pos is global position within this element block...
2255 global_block_pos = local_element_to_global[p][(e + boffset)] - goffset;
2256 global_pos = global_block_pos * att_count;
2257 for (size_t n = 0; n < att_count; n++) {
2258 attributes[global_pos++] = local_attr[pos++];
2259 }
2260 }
2261 }
2262
2263 } // end if blocks[p][b].entity_count() (non-zero length block)
2264 else if (debug_level & 4) {
2265 fmt::print(".");
2266 }
2267 } // end for p=0..part_count-1
2268
2269 // Write out block info
2270 int id_out = ExodusFile::output(); // output file identifier
2271
2272 if (!linkage.empty()) {
2273 error =
2274 ex_put_conn(id_out, EX_ELEM_BLOCK, glob_blocks[b].id, Data(linkage), nullptr, nullptr);
2275 if (error < 0) {
2276 exodus_error(__LINE__);
2277 }
2278 }
2279
2280 // Write out attributes list if it exists
2281 if (glob_blocks[b].attributeCount > 0) {
2282 error = ex_put_attr(id_out, EX_ELEM_BLOCK, glob_blocks[b].id, Data(attributes));
2283 if (error < 0) {
2284 exodus_error(__LINE__);
2285 }
2286 } // end for b=0..global_num_blocks-1
2287 if (debug_level & 4) {
2288 fmt::print("\n");
2289 }
2290 }
2291 fmt::print("\n");
2292 }
2293
2294 template <typename T, typename INT>
2295 void put_element_blocks(int part_count, int start_part, std::vector<std::vector<Block>> &blocks,
2296 std::vector<Block> &glob_blocks,
2297 const std::vector<std::vector<INT>> &local_node_to_global)
2298 {
2299 // This variant of `put_element_blocks` is used in the case of
2300 // `nomap` and uses much less memory (but may be slower). It
2301 // relies on the fact that with `nomap`, a the elements for a
2302 // block in a parts are all contiguous in the output file, so we
2303 // can read them, map the nodes, and then output them using a
2304 // partial write function and they don't need to be mapped into a
2305 // global array (which also does not need to be allocated). Phase
2306 // 1 of a "low-memory" option for epu.
2307
2308 SMART_ASSERT(sizeof(T) == ExodusFile::io_word_size());
2309 int global_num_blocks = glob_blocks.size();
2310
2311 LOG("\nReading and Writing element connectivity & attributes (Low Memory Method)\n");
2312
2313 for (int b = 0; b < global_num_blocks; b++) {
2314
2315 if (debug_level & 4) {
2316 fmt::print("\nOutput element block info for...\n"
2317 "Block {}, Id = {}, Name = '{}', Elements = {:12}, Nodes/element = {}, "
2318 "Attributes = {}\n",
2319 b, glob_blocks[b].id, glob_blocks[b].name_,
2320 fmt::group_digits(glob_blocks[b].entity_count()), glob_blocks[b].nodesPerElement,
2321 glob_blocks[b].attributeCount);
2322 }
2323
2324 int id_out = ExodusFile::output(); // output file identifier
2325
2326 size_t part_block_offset = 1;
2327 for (int p = 0; p < part_count; p++) {
2328 ExodusFile id(p);
2329
2330 if (blocks[p][b].entity_count() > 0) { // non-zero length block
2331 size_t node_count = blocks[p][b].entity_count() * blocks[p][b].nodesPerElement;
2332 std::vector<INT> local_linkage(node_count);
2333
2334 ex_entity_id bid = blocks[p][b].id;
2335 int error = ex_get_conn(id, EX_ELEM_BLOCK, bid, Data(local_linkage), nullptr, nullptr);
2336 if (error < 0) {
2337 fmt::print(
2338 stderr,
2339 "ERROR: (EPU) Cannot get element block connectivity for block {} on part {}.\n",
2340 bid, p + start_part);
2341 exodus_error(__LINE__);
2342 }
2343
2344 size_t element_count = blocks[p][b].entity_count();
2345 size_t npe = blocks[p][b].nodesPerElement;
2346 const std::vector<INT> &proc_loc_node_to_global = local_node_to_global[p];
2347
2348 size_t pos = 0;
2349 for (size_t e = 0; e < element_count; e++) {
2350 for (size_t n = 0; n < npe; n++) {
2351 size_t node = proc_loc_node_to_global[local_linkage[pos] - 1];
2352 local_linkage[pos++] = node + 1;
2353 }
2354 }
2355
2356 if (debug_level & 4) {
2357 fmt::print(stderr, "part, block, offset, count = {} {} {} {}\n", p, bid,
2358 part_block_offset, fmt::group_digits(element_count));
2359 }
2360 error = ex_put_partial_conn(id_out, EX_ELEM_BLOCK, bid, part_block_offset, element_count,
2361 Data(local_linkage), nullptr, nullptr);
2362 if (error < 0) {
2363 fmt::print(stderr,
2364 "ERROR: (EPU) Cannot output element block connectivity for block {} on part "
2365 "{} (offset = {}, count = {}).\n",
2366 bid, p + start_part, part_block_offset, fmt::group_digits(element_count));
2367 exodus_error(__LINE__);
2368 }
2369
2370 // Get attributes list, if it exists
2371 if (blocks[p][b].attributeCount > 0) {
2372 size_t max_attr = blocks[p][b].entity_count() * blocks[p][b].attributeCount;
2373 std::vector<T> local_attr(max_attr);
2374
2375 error = ex_get_attr(id, EX_ELEM_BLOCK, blocks[p][b].id, Data(local_attr));
2376 if (error < 0) {
2377 exodus_error(__LINE__);
2378 }
2379
2380 error = ex_put_partial_attr(id_out, EX_ELEM_BLOCK, bid, part_block_offset,
2381 element_count, Data(local_attr));
2382 if (error < 0) {
2383 exodus_error(__LINE__);
2384 }
2385 }
2386 part_block_offset += element_count;
2387
2388 } // end if blocks[p][b].entity_count() (non-zero length block)
2389 } // end for p=0..part_count-1
2390 }
2391 }
2392
2393 template <typename INT>
2394 void build_reverse_element_map(std::vector<std::vector<INT>> &local_element_to_global,
2395 const std::vector<Mesh> &local_mesh,
2396 std::vector<std::vector<Block>> &blocks,
2397 std::vector<Block> &glob_blocks, Mesh *global, int part_count,
2398 std::vector<INT> &global_element_map, bool map_ids)
2399 {
2400 // Global element map and count.
2401 std::vector<std::vector<INT>> global_element_numbers(part_count);
2402
2403 size_t tot_size = 0;
2404 for (int p = 0; p < part_count; p++) {
2405 tot_size += local_mesh[p].elementCount;
2406 global_element_numbers[p].resize(local_mesh[p].elementCount);
2407 }
2408 global_element_map.resize(tot_size);
2409
2410 {
2411 size_t offset = 0;
2412 for (int p = 0; p < part_count; p++) {
2413 ExodusFile id(p);
2414 get_id_map(id, EX_ELEM_MAP, EX_INQ_ELEM_MAP, global_element_numbers[p]);
2415 std::copy(global_element_numbers[p].begin(), global_element_numbers[p].end(),
2416 &global_element_map[offset]);
2417 offset += local_mesh[p].elementCount;
2418 }
2419 }
2420
2421 // Now, sort the global_element_map array.
2422 #if USE_STD_SORT
2423 std::sort(global_element_map.begin(), global_element_map.end());
2424 #else
2425 pdqsort(global_element_map.begin(), global_element_map.end());
2426 #endif
2427
2428 global->elementCount = global_element_map.size();
2429
2430 // See if any duplicates...
2431 for (int64_t i = 1; i < global->elementCount; i++) {
2432 if (global_element_map[i - 1] == global_element_map[i]) {
2433 // Duplicates in the element id list...
2434 // This is not yet handled. Notify the user and continue for now...
2435 fmt::print(stderr, "\n!!!! POSSIBLE ERROR: (EPU) There were at least 2"
2436 " elements with duplicated ids detected.\n"
2437 "!!!!\tThis may cause problems in the output file.\n\n");
2438 break;
2439 }
2440 }
2441
2442 // See whether the element numbers are contiguous. If so, we can map
2443 // the elements back to their original location. Since the elements are
2444 // sorted and there are no duplicates, we just need to see if the id
2445 // at global_element_map.size() == global_element_map.size();
2446 bool is_contiguous = global_element_map.empty() ||
2447 ((size_t)global_element_map.back() == global_element_map.size());
2448 if (rank == 0) {
2449 fmt::print("Element id map {} contiguous.\n", (is_contiguous ? "is" : "is not"));
2450 }
2451
2452 // Create the map that maps from a local processor element to the
2453 // global map. This combines the mapping local processor element to
2454 // 'global id' and then 'global id' to global position. The
2455 // mapping is now a direct lookup instead of a lookup followed by
2456 // a reverse map.
2457 //
2458 // If the map is contiguous, then the global_id to global_position map is 1->1
2459 REMAP:
2460 if (is_contiguous && map_ids) {
2461 auto cur_pos = global_element_map.begin();
2462 size_t element_value;
2463 for (int p = 0; p < part_count; p++) {
2464 size_t element_count = local_mesh[p].elementCount;
2465 for (size_t i = 0; i < element_count; i++) {
2466 INT global_element = global_element_numbers[p][i];
2467
2468 if (cur_pos == global_element_map.end() || *cur_pos != global_element) {
2469 auto iter = std::lower_bound(global_element_map.begin(), global_element_map.end(),
2470 global_element);
2471 SMART_ASSERT(iter != global_element_map.end());
2472 cur_pos = iter;
2473 }
2474 element_value = cur_pos - global_element_map.begin();
2475 local_element_to_global[p][i] = element_value;
2476 ++cur_pos;
2477 }
2478 }
2479 }
2480 else {
2481 IntVector proc_off(part_count);
2482 std::fill(proc_off.begin(), proc_off.end(), 0);
2483
2484 size_t gpos = 0;
2485 for (size_t b = 0; b < glob_blocks.size(); b++) {
2486 for (int p = 0; p < part_count; p++) {
2487 size_t poff = proc_off[p];
2488 size_t element_count = blocks[p][b].entity_count();
2489 for (size_t e = 0; e < element_count; e++) {
2490 local_element_to_global[p][e + poff] = gpos++;
2491 }
2492 proc_off[p] += element_count;
2493 }
2494 }
2495
2496 std::vector<INT> element_map;
2497 for (int p = 0; p < part_count; p++) {
2498 size_t element_count = local_mesh[p].elementCount;
2499 element_map.resize(element_count);
2500 get_id_map(ExodusFile(p), EX_ELEM_MAP, EX_INQ_ELEM_MAP, element_map);
2501
2502 for (size_t e = 0; e < element_count; e++) {
2503 gpos = local_element_to_global[p][e];
2504 global_element_map[gpos] = element_map[e];
2505 }
2506 }
2507 }
2508
2509 // Now, need to set up the element block offsets. Within an element
2510 // block, the ids are contiguous, so to map a global element to its
2511 // position within the element block, use the formula:
2512 //
2513 // pos_in_block = global_pos - block_offset
2514 //
2515 // Since there is the possibility that the element blocks in this
2516 // file are in a different order than the element blocks in original
2517 // file, we need to find out the minimum element id in each element
2518 // block and that is the offset (0-based ids). Will also get the
2519 // maximum id to use as a sanity check.
2520 // Note that ids are contiguous from 0..element_count-1
2521
2522 for (size_t b = 0; b < glob_blocks.size(); b++) {
2523 size_t min_id = global_element_map.size();
2524 size_t max_id = 0;
2525
2526 for (int p = 0; p < part_count; p++) {
2527 size_t offset = blocks[p][b].offset_;
2528 size_t element_count = blocks[p][b].entity_count();
2529 for (size_t e = 0; e < element_count; e++) {
2530 size_t id = local_element_to_global[p][e + offset];
2531 min_id = (id < min_id) ? id : min_id;
2532 max_id = (id > max_id) ? id : max_id;
2533 }
2534 }
2535
2536 if (glob_blocks[b].entity_count() == 0) {
2537 min_id = 0;
2538 max_id = 0;
2539 }
2540 else {
2541 if (max_id - min_id + 1 != glob_blocks[b].entity_count()) {
2542 if (map_ids) {
2543 map_ids = false;
2544 fmt::print(stderr,
2545 "WARNING: The element ids are globally contiguous,\n"
2546 "\tbut they are not consistent for element block {}.\n"
2547 "\tRetrying with element id mapping turned off.\n",
2548 glob_blocks[b].id);
2549 goto REMAP;
2550 }
2551 else {
2552 std::ostringstream errmsg;
2553 fmt::print(errmsg,
2554 "ERROR: (EPU) The element ids for element block {} are not consistent.\n"
2555 "Block {}, Id = {} min/max id = {}/{} size = {}.\n",
2556 glob_blocks[b].id, b, glob_blocks[b].id, min_id + 1, max_id + 1,
2557 fmt::group_digits(glob_blocks[b].entity_count()));
2558 throw std::runtime_error(errmsg.str());
2559 }
2560 }
2561 }
2562 glob_blocks[b].offset_ = min_id;
2563 if (debug_level & 4) {
2564 fmt::print("Block {}, Id = {} min/max id = {}/{} offset = {}\n", b, glob_blocks[b].id,
2565 min_id + 1, max_id + 1, glob_blocks[b].offset_);
2566 }
2567 }
2568 }
2569
2570 template <typename INT>
2571 void build_reverse_edge_map(std::vector<std::vector<INT>> &local_edge_to_global,
2572 const std::vector<Mesh> &local_mesh,
2573 std::vector<std::vector<EdgeBlock<INT>>> &edgeblocks,
2574 std::vector<EdgeBlock<INT>> &glob_edgeblocks, Mesh *global,
2575 int part_count, std::vector<INT> &global_edge_map, bool map_ids)
2576 {
2577 // Global edge map and count
2578 std::vector<std::vector<INT>> global_edge_numbers(part_count);
2579
2580 size_t tot_size = 0;
2581 for (int p = 0; p < part_count; p++) {
2582 tot_size += local_mesh[p].edgeCount;
2583 global_edge_numbers[p].resize(local_mesh[p].edgeCount);
2584 }
2585 global_edge_map.resize(tot_size);
2586
2587 {
2588 size_t offset = 0;
2589 for (int p = 0; p < part_count; p++) {
2590 ExodusFile id(p);
2591 get_id_map(id, EX_EDGE_MAP, EX_INQ_EDGE_MAP, global_edge_numbers[p]);
2592 std::copy(global_edge_numbers[p].begin(), global_edge_numbers[p].end(),
2593 &global_edge_map[offset]);
2594 offset += local_mesh[p].edgeCount;
2595 }
2596 }
2597
2598 // Now, sort the global_edge_map array.
2599 #if USE_STD_SORT
2600 std::sort(global_edge_map.begin(), global_edge_map.end());
2601 #else
2602 pdqsort(global_edge_map.begin(), global_edge_map.end());
2603 #endif
2604
2605 global->edgeCount = global_edge_map.size();
2606
2607 // See if any duplicates...
2608 for (int64_t i = 1; i < global->edgeCount; i++) {
2609 if (global_edge_map[i - 1] == global_edge_map[i]) {
2610 // Duplicates in the edge id list...
2611 // This is not yet handled. Notify the user and continue for now...
2612 fmt::print(stderr, "\n!!!! POSSIBLE ERROR: (EPU) There were at least 2"
2613 " edges with duplicated ids detected.\n"
2614 "!!!!\tThis may cause problems in the output file.\n\n");
2615 break;
2616 }
2617 }
2618
2619 // See whether the edge numbers are contiguous. If so, we can map
2620 // the edges back to their original location. Since the edges are
2621 // sorted and there are no duplicates, we just need to see if the id
2622 // at global_edge_map.size() == global_edge_map.size();
2623 bool is_contiguous =
2624 global_edge_map.empty() || ((size_t)global_edge_map.back() == global_edge_map.size());
2625 if (rank == 0) {
2626 fmt::print("Edge id map {} contiguous.\n", (is_contiguous ? "is" : "is not"));
2627 }
2628
2629 // Create the map that maps from a local processor edge to the
2630 // global map. This combines the mapping local processor edge to
2631 // 'global id' and then 'global id' to global position. The
2632 // mapping is now a direct lookup instead of a lookup followed by
2633 // a reverse map.
2634 //
2635 // If the map is contiguous, then the global_id to global_position map is 1->1
2636 REMAP:
2637 if (is_contiguous && map_ids) {
2638 auto cur_pos = global_edge_map.begin();
2639 size_t edge_value;
2640 for (int p = 0; p < part_count; p++) {
2641 size_t edge_count = local_mesh[p].edgeCount;
2642 for (size_t i = 0; i < edge_count; i++) {
2643 INT global_edge = global_edge_numbers[p][i];
2644
2645 if (cur_pos == global_edge_map.end() || *cur_pos != global_edge) {
2646 auto iter =
2647 std::lower_bound(global_edge_map.begin(), global_edge_map.end(), global_edge);
2648 SMART_ASSERT(iter != global_edge_map.end());
2649 cur_pos = iter;
2650 }
2651 edge_value = cur_pos - global_edge_map.begin();
2652 local_edge_to_global[p][i] = edge_value;
2653 ++cur_pos;
2654 }
2655 }
2656 }
2657 else {
2658 IntVector proc_off(part_count);
2659 std::fill(proc_off.begin(), proc_off.end(), 0);
2660
2661 size_t gpos = 0;
2662 for (size_t b = 0; b < glob_edgeblocks.size(); b++) {
2663 for (int p = 0; p < part_count; p++) {
2664 size_t poff = proc_off[p];
2665 size_t edge_count = edgeblocks[p][b].entity_count();
2666 for (size_t e = 0; e < edge_count; e++) {
2667 local_edge_to_global[p][e + poff] = gpos++;
2668 }
2669 proc_off[p] += edge_count;
2670 }
2671 }
2672
2673 std::vector<INT> edge_map;
2674 for (int p = 0; p < part_count; p++) {
2675 size_t edge_count = local_mesh[p].edgeCount;
2676 edge_map.resize(edge_count);
2677 get_id_map(ExodusFile(p), EX_EDGE_MAP, EX_INQ_EDGE_MAP, edge_map);
2678
2679 for (size_t e = 0; e < edge_count; e++) {
2680 gpos = local_edge_to_global[p][e];
2681 global_edge_map[gpos] = edge_map[e];
2682 }
2683 }
2684 }
2685
2686 // Now, need to set up the edge block offsets. Within an edge
2687 // block, the ids are contiguous, so to map a global edge to its
2688 // position within the edge block, use the formula:
2689 //
2690 // pos_in_block = global_pos - block_offset
2691 //
2692 // Since there is the possibility that the edge blocks in this
2693 // file are in a different order than the edge blocks in original
2694 // file, we need to find out the minimum edge id in each edge
2695 // block and that is the offset (0-based ids). Will also get the
2696 // maximum id to use as a sanity check.
2697 // Note that ids are contiguous from 0..edge_count-1
2698
2699 for (size_t b = 0; b < glob_edgeblocks.size(); b++) {
2700 size_t min_id = global_edge_map.size();
2701 size_t max_id = 0;
2702
2703 for (int p = 0; p < part_count; p++) {
2704 size_t offset = edgeblocks[p][b].offset_;
2705 size_t edge_count = edgeblocks[p][b].entity_count();
2706 for (size_t e = 0; e < edge_count; e++) {
2707 size_t id = local_edge_to_global[p][e + offset];
2708 min_id = (id < min_id) ? id : min_id;
2709 max_id = (id > max_id) ? id : max_id;
2710 }
2711 }
2712
2713 if (glob_edgeblocks[b].entity_count() == 0) {
2714 min_id = 0;
2715 max_id = 0;
2716 }
2717 else {
2718 if (max_id - min_id + 1 != glob_edgeblocks[b].entity_count()) {
2719 if (map_ids) {
2720 map_ids = false;
2721 fmt::print(stderr,
2722 "WARNING: The edge ids are globally contiguous,\n"
2723 "\tbut they are not consistent for edge block {}.\n"
2724 "\tRetrying with edge id mapping turned off.\n",
2725 glob_edgeblocks[b].id);
2726 goto REMAP;
2727 }
2728 else {
2729 std::ostringstream errmsg;
2730 fmt::print(errmsg,
2731 "ERROR: (EPU) The edge ids for edge block {} are not consistent.\n"
2732 "Block {}, Id = {} min/max id = {}/{} size = {}.\n",
2733 glob_edgeblocks[b].id, b, glob_edgeblocks[b].id, min_id + 1, max_id + 1,
2734 fmt::group_digits(glob_edgeblocks[b].entity_count()));
2735 throw std::runtime_error(errmsg.str());
2736 }
2737 }
2738 }
2739 glob_edgeblocks[b].offset_ = min_id;
2740 if (debug_level & 4) {
2741 fmt::print("Block {}, Id = {} min/max id = {}/{} offset = {}\n", b, glob_edgeblocks[b].id,
2742 min_id + 1, max_id + 1, glob_edgeblocks[b].offset_);
2743 }
2744 }
2745 }
2746
2747 template <typename INT>
2748 void build_reverse_face_map(std::vector<std::vector<INT>> &local_face_to_global,
2749 const std::vector<Mesh> &local_mesh,
2750 std::vector<std::vector<FaceBlock<INT>>> &faceblocks,
2751 std::vector<FaceBlock<INT>> &glob_faceblocks, Mesh *global,
2752 int part_count, std::vector<INT> &global_face_map, bool map_ids)
2753 {
2754 // Global element map and count.
2755 std::vector<std::vector<INT>> global_face_numbers(part_count);
2756
2757 size_t tot_size = 0;
2758 for (int p = 0; p < part_count; p++) {
2759 tot_size += local_mesh[p].faceCount;
2760 global_face_numbers[p].resize(local_mesh[p].faceCount);
2761 }
2762 global_face_map.resize(tot_size);
2763
2764 {
2765 size_t offset = 0;
2766 for (int p = 0; p < part_count; p++) {
2767 ExodusFile id(p);
2768 get_id_map(id, EX_FACE_MAP, EX_INQ_FACE_MAP, global_face_numbers[p]);
2769 std::copy(global_face_numbers[p].begin(), global_face_numbers[p].end(),
2770 &global_face_map[offset]);
2771 offset += local_mesh[p].faceCount;
2772 }
2773 }
2774
2775 // Now, sort the global_face_map array.
2776 #if USE_STD_SORT
2777 std::sort(global_face_map.begin(), global_face_map.end());
2778 #else
2779 pdqsort(global_face_map.begin(), global_face_map.end());
2780 #endif
2781
2782 global->faceCount = global_face_map.size();
2783
2784 // See if any duplicates...
2785 for (int64_t i = 1; i < global->faceCount; i++) {
2786 if (global_face_map[i - 1] == global_face_map[i]) {
2787 // Duplicates in the face id list...
2788 // This is not yet handled. Notify the user and continue for now...
2789 fmt::print(stderr, "\n!!!! POSSIBLE ERROR: (EPU) There were at least 2"
2790 " faces with duplicated ids detected.\n"
2791 "!!!!\tThis may cause problems in the output file.\n\n");
2792 break;
2793 }
2794 }
2795
2796 // See whether the face numbers are contiguous. If so, we can map
2797 // the faces back to their original location. Since the faces are
2798 // sorted and there are no duplicates, we just need to see if the id
2799 // at global_face_map.size() == global_face_map.size();
2800 bool is_contiguous =
2801 global_face_map.empty() || ((size_t)global_face_map.back() == global_face_map.size());
2802 if (rank == 0) {
2803 fmt::print("Face id map {} contiguous.\n", (is_contiguous ? "is" : "is not"));
2804 }
2805
2806 // Create the map that maps from a local processor face to the
2807 // global map. This combines the mapping local processor face to
2808 // 'global id' and then 'global id' to global position. The
2809 // mapping is now a direct lookup instead of a lookup followed by
2810 // a reverse map.
2811 //
2812 // If the map is contiguous, then the global_id to global_position map is 1->1
2813 REMAP:
2814 if (is_contiguous && map_ids) {
2815 auto cur_pos = global_face_map.begin();
2816 size_t face_value;
2817 for (int p = 0; p < part_count; p++) {
2818 size_t face_count = local_mesh[p].faceCount;
2819 for (size_t i = 0; i < face_count; i++) {
2820 INT global_face = global_face_numbers[p][i];
2821
2822 if (cur_pos == global_face_map.end() || *cur_pos != global_face) {
2823 auto iter =
2824 std::lower_bound(global_face_map.begin(), global_face_map.end(), global_face);
2825 SMART_ASSERT(iter != global_face_map.end());
2826 cur_pos = iter;
2827 }
2828 face_value = cur_pos - global_face_map.begin();
2829 local_face_to_global[p][i] = face_value;
2830 ++cur_pos;
2831 }
2832 }
2833 }
2834 else {
2835 IntVector proc_off(part_count);
2836 std::fill(proc_off.begin(), proc_off.end(), 0);
2837
2838 size_t gpos = 0;
2839 for (size_t b = 0; b < glob_faceblocks.size(); b++) {
2840 for (int p = 0; p < part_count; p++) {
2841 size_t poff = proc_off[p];
2842 size_t face_count = faceblocks[p][b].entity_count();
2843 for (size_t e = 0; e < face_count; e++) {
2844 local_face_to_global[p][e + poff] = gpos++;
2845 }
2846 proc_off[p] += face_count;
2847 }
2848 }
2849
2850 std::vector<INT> face_map;
2851 for (int p = 0; p < part_count; p++) {
2852 size_t face_count = local_mesh[p].faceCount;
2853 face_map.resize(face_count);
2854 get_id_map(ExodusFile(p), EX_ELEM_MAP, EX_INQ_ELEM_MAP, face_map);
2855
2856 for (size_t e = 0; e < face_count; e++) {
2857 gpos = local_face_to_global[p][e];
2858 global_face_map[gpos] = face_map[e];
2859 }
2860 }
2861 }
2862
2863 // Now, need to set up the face block offsets. Within an face
2864 // block, the ids are contiguous, so to map a global face to its
2865 // position within the face block, use the formula:
2866 //
2867 // pos_in_block = global_pos - block_offset
2868 //
2869 // Since there is the possibility that the face blocks in this
2870 // file are in a different order than the face blocks in original
2871 // file, we need to find out the minimum face id in each face
2872 // block and that is the offset (0-based ids). Will also get the
2873 // maximum id to use as a sanity check.
2874 // Note that ids are contiguous from 0..face_count-1
2875
2876 for (size_t b = 0; b < glob_faceblocks.size(); b++) {
2877 size_t min_id = global_face_map.size();
2878 size_t max_id = 0;
2879
2880 for (int p = 0; p < part_count; p++) {
2881 size_t offset = faceblocks[p][b].offset_;
2882 size_t face_count = faceblocks[p][b].entity_count();
2883 for (size_t e = 0; e < face_count; e++) {
2884 size_t id = local_face_to_global[p][e + offset];
2885 min_id = (id < min_id) ? id : min_id;
2886 max_id = (id > max_id) ? id : max_id;
2887 }
2888 }
2889
2890 if (glob_faceblocks[b].entity_count() == 0) {
2891 min_id = 0;
2892 max_id = 0;
2893 }
2894 else {
2895 if (max_id - min_id + 1 != glob_faceblocks[b].entity_count()) {
2896 if (map_ids) {
2897 map_ids = false;
2898 fmt::print(stderr,
2899 "WARNING: The face ids are globally contiguous,\n"
2900 "\tbut they are not consistent for face block {}.\n"
2901 "\tRetrying with face id mapping turned off.\n",
2902 glob_faceblocks[b].id);
2903 goto REMAP;
2904 }
2905 else {
2906 std::ostringstream errmsg;
2907 fmt::print(errmsg,
2908 "ERROR: (EPU) The face ids for face block {} are not consistent.\n"
2909 "Block {}, Id = {} min/max id = {}/{} size = {}.\n",
2910 glob_faceblocks[b].id, b, glob_faceblocks[b].id, min_id + 1, max_id + 1,
2911 fmt::group_digits(glob_faceblocks[b].entity_count()));
2912 throw std::runtime_error(errmsg.str());
2913 }
2914 }
2915 }
2916 glob_faceblocks[b].offset_ = min_id;
2917 if (debug_level & 4) {
2918 fmt::print("Block {}, Id = {} min/max id = {}/{} offset = {}\n", b, glob_faceblocks[b].id,
2919 min_id + 1, max_id + 1, glob_faceblocks[b].offset_);
2920 }
2921 }
2922 }
2923
2924 template <typename INT>
2925 void build_reverse_node_map(std::vector<std::vector<INT>> &local_node_to_global,
2926 const std::vector<Mesh> &local_mesh, Mesh *global, int part_count,
2927 std::vector<INT> &global_node_map)
2928 {
2929 // Instead of using <set> and <map>, consider using a sorted vector...
2930 // Append all local node maps to the global node map.
2931 // Sort the global node map
2932 // Remove duplicates.
2933 // Position within map is now the map...
2934 // When building the local-proc node to global id, use binary_search...
2935
2936 // Global node map and count.
2937 std::vector<std::vector<INT>> global_node_numbers(part_count);
2938
2939 size_t tot_size = 0;
2940 for (int p = 0; p < part_count; p++) {
2941 tot_size += local_mesh[p].nodeCount;
2942 global_node_numbers[p].resize(local_mesh[p].nodeCount);
2943 }
2944 global_node_map.resize(tot_size);
2945
2946 size_t offset = 0;
2947 for (int p = 0; p < part_count; p++) {
2948 ExodusFile id(p);
2949 get_id_map(id, EX_NODE_MAP, EX_INQ_NODE_MAP, global_node_numbers[p]);
2950 std::copy(global_node_numbers[p].begin(), global_node_numbers[p].end(),
2951 &global_node_map[offset]);
2952 offset += local_mesh[p].nodeCount;
2953 }
2954
2955 // Now, sort the global_node_map array and remove duplicates...
2956 #if USE_STD_SORT
2957 std::sort(global_node_map.begin(), global_node_map.end());
2958 #else
2959 pdqsort(global_node_map.begin(), global_node_map.end());
2960 #endif
2961 global_node_map.resize(unique(global_node_map));
2962 global_node_map.shrink_to_fit();
2963
2964 size_t total_num_nodes = global_node_map.size();
2965 global->nodeCount = total_num_nodes;
2966
2967 // See whether the node numbers are contiguous. If so, we can map
2968 // the nodes back to their original location. Since the nodes are
2969 // sorted and there are no duplicates, we just need to see if the id
2970 // at global_node_map.size() == global_node_map.size();
2971 bool is_contiguous =
2972 global_node_map.empty() || ((size_t)global_node_map.back() == global_node_map.size());
2973 if (rank == 0) {
2974 fmt::print("Node map {} contiguous.\n", (is_contiguous ? "is" : "is not"));
2975 }
2976
2977 // Create the map the maps from a local processor node to the
2978 // global map. This combines the mapping local processor node to
2979 // 'global id' and then 'global id' to global position. The
2980 // mapping is now a direct lookup instead of a lookup followed by
2981 // a reverse map.
2982 auto cur_pos = global_node_map.begin();
2983 INT nodal_value;
2984 for (int p = 0; p < part_count; p++) {
2985 size_t node_count = local_mesh[p].nodeCount;
2986 for (size_t i = 0; i < node_count; i++) {
2987 INT global_node = global_node_numbers[p][i];
2988
2989 if (cur_pos == global_node_map.end() || *cur_pos != global_node) {
2990 auto iter = std::lower_bound(global_node_map.begin(), global_node_map.end(), global_node);
2991 SMART_ASSERT(iter != global_node_map.end());
2992 cur_pos = iter;
2993 }
2994 nodal_value = cur_pos - global_node_map.begin();
2995 local_node_to_global[p][i] = nodal_value;
2996 ++cur_pos;
2997 }
2998 }
2999 }
3000
3001 void get_put_variable_names(int id, int out, Variables &vars, Excn::SystemInterface &interFace)
3002 {
3003 if (vars.count(InOut::OUT) > 0) {
3004
3005 char **output_name_list =
3006 get_name_array(vars.count(InOut::OUT), ExodusFile::max_name_length());
3007
3008 int extra = vars.add_processor_id() ? 1 : 0;
3009 int num_input_vars = vars.index_.size();
3010
3011 char **input_name_list = get_name_array(num_input_vars, ExodusFile::max_name_length());
3012 int error = ex_get_variable_names(id, vars.type(), num_input_vars - extra, input_name_list);
3013 if (error < 0) {
3014 exodus_error(__LINE__);
3015 }
3016
3017 if (vars.add_processor_id()) {
3018 // Check that input list of names does not already contain 'processor_id'...
3019 // If found, create an 'epu'-specific name; don't redo the check; assume ok...
3020 bool found = false;
3021 for (int i = 0; i < num_input_vars && !found; i++) {
3022 if (case_compare(input_name_list[i], "processor_id")) {
3023 found = true;
3024 }
3025 }
3026 if (found) {
3027 fmt::print(stderr, "\nWARNING: Variable 'processor_id' already exists on database.\n"
3028 " Adding 'processor_id_epu' instead.\n\n");
3029 copy_string(input_name_list[num_input_vars - 1], "processor_id_epu",
3030 ExodusFile::max_name_length() + 1);
3031 }
3032 else {
3033 copy_string(input_name_list[num_input_vars - 1], "processor_id",
3034 ExodusFile::max_name_length() + 1);
3035 }
3036 }
3037
3038 // Iterate through the 'var_index' and transfer
3039 // Assume that the number of pointers is limited to
3040 // the number of results variables, plus one
3041 // extra pointer for optional add_processor_id
3042 size_t maxlen = 0;
3043 for (int i = 0; i < num_input_vars; i++) {
3044 if (vars.index_[i] > 0) {
3045 copy_string(output_name_list[vars.index_[i] - 1], input_name_list[i],
3046 ExodusFile::max_name_length() + 1);
3047 if (strlen(input_name_list[i]) > maxlen) {
3048 maxlen = strlen(input_name_list[i]);
3049 }
3050 }
3051 }
3052 maxlen += 2;
3053 int width = interFace.screen_width();
3054 // Assume 8 characters for initial tab...
3055 int nfield = (width - 8) / maxlen;
3056 if (nfield < 1) {
3057 nfield = 1;
3058 }
3059
3060 if (rank == 0) {
3061 fmt::print("Found {} {} variables.\n\t", vars.count(InOut::OUT), vars.label());
3062 int i = 0;
3063 int ifld = 1;
3064 while (i < vars.count(InOut::OUT)) {
3065 fmt::print("{:<{}}", output_name_list[i++], maxlen);
3066 if (++ifld > nfield && i < vars.count(InOut::OUT)) {
3067 fmt::print("\n\t");
3068 ifld = 1;
3069 }
3070 }
3071 fmt::print("\n\n");
3072 }
3073
3074 if (!interFace.append()) {
3075 error = ex_put_variable_names(out, vars.type(), vars.count(InOut::OUT), output_name_list);
3076 if (error < 0) {
3077 exodus_error(__LINE__);
3078 }
3079 }
3080
3081 free_name_array(output_name_list, vars.count(InOut::OUT));
3082 free_name_array(input_name_list, num_input_vars);
3083 }
3084 }
3085
3086 void get_variable_params(int id, Variables &vars, const StringIdVector &variable_list)
3087 {
3088 // Determines the number of variables of type 'type()' that will
3089 // be written to the output database. The 'variable_list' vector
3090 // specifies a possibly empty list of variable names that the user
3091 // wants transferred to the output database. If 'variable_list' is
3092 // empty, then all variables of that type will be transferred; if
3093 // the 'variable_list' size is 1 and it contains the string 'NONE',
3094 // then no variables of that type will be transferred; if size is 1
3095 // and it contains the string 'ALL', then all variables of that type
3096 // will be transferred.
3097 //
3098 // Returns the number of variables which will be output Also creates
3099 // a 'var_index'. The var_index is zero-based and of size
3100 // 'input_variable_count'. If:
3101 // var_index[i] ==0, variable not written to output database
3102 // var_index[i] > 0, variable written; is variable 'var_index[i]'
3103
3104 // If 'type' is ELEMENT and addProcessorId is true, then reserve
3105 // space for an additional variable.
3106 int extra = 0;
3107 if (vars.type() == EX_ELEM_BLOCK && vars.addProcessorId) {
3108 extra = 1;
3109 }
3110
3111 int num_vars;
3112 int error = ex_get_variable_param(id, vars.type(), &num_vars);
3113 if (error < 0) {
3114 exodus_error(__LINE__);
3115 }
3116
3117 vars.index_.resize(num_vars + extra);
3118
3119 // Create initial index which defaults to no output...
3120 std::fill(vars.index_.begin(), vars.index_.end(), 0);
3121 // ...Except for the processor_id (if any)
3122 if (vars.addProcessorId) {
3123 vars.index_[num_vars] = 1;
3124 }
3125
3126 // If 'variable_list' is empty or specified 'ALL', then all
3127 // variables are to be output
3128 if (variable_list.empty() ||
3129 (variable_list.size() == 1 && case_compare(variable_list[0].first, "all"))) {
3130 for (size_t i = 0; i < vars.index_.size(); i++) {
3131 vars.index_[i] = i + 1;
3132 }
3133 vars.outputCount = num_vars + extra;
3134 return;
3135 }
3136
3137 // Another possibility is user specifies "NONE" for the variable
3138 // list so no variables will be written. Just return 0 or 1 based
3139 // on the 'add_processor_id' setting. Default var_index is ok.
3140 if (variable_list.size() == 1 && case_compare(variable_list[0].first, "none")) {
3141 vars.outputCount = extra;
3142 return;
3143 }
3144
3145 // At this point, the variable_list specifies at least one
3146 // variable to be output to the database.
3147 // Get the list that the user entered and the list of files
3148 // from the input database...
3149 {
3150 StringVector exo_names = get_exodus_variable_names(id, vars.type(), num_vars);
3151
3152 // Iterate 'variable_list' and find position in 'exo_names'. If
3153 // not found, there is an error -- set index to -1; otherwise set
3154 // index to position (1-based) in variable_list. Others are
3155 // already set to '0' by default initialization.
3156
3157 // The variable_list may contain multiple entries for each
3158 // variable if the user is specifying output only on certain
3159 // element blocks...
3160 std::string var_name;
3161 int var_count = 0;
3162 for (const auto &elem : variable_list) {
3163 if (var_name == elem.first) {
3164 continue;
3165 }
3166 var_name = elem.first;
3167 bool found = false;
3168 for (size_t j = 0; j < exo_names.size() && !found; j++) {
3169 if (case_compare(exo_names[j], var_name)) {
3170 found = true;
3171 vars.index_[j] = ++var_count;
3172 }
3173 }
3174 if (!found) {
3175 std::ostringstream errmsg;
3176 fmt::print(errmsg, "ERROR: (EPU) Variable '{}' is not valid.\n", elem.first);
3177 throw std::runtime_error(errmsg.str());
3178 }
3179 }
3180 // Count non-zero entries in var_index;
3181 int nz_count = 0;
3182 for (auto &elem : vars.index_) {
3183 if (elem > 0) {
3184 nz_count++;
3185 }
3186 }
3187 SMART_ASSERT(nz_count == var_count + extra)(nz_count)(var_count);
3188
3189 if (vars.addProcessorId) {
3190 vars.index_[num_vars] = nz_count; // Already counted above...
3191 }
3192 vars.outputCount = nz_count;
3193 return;
3194 }
3195 }
3196
3197 void put_global_info(const Mesh &global)
3198 {
3199 // Write out Global info
3200 if (rank == 0) {
3201 fmt::print(" Title: {}\n\n"
3202 " Number of coordinates per node = {:15}\n"
3203 " Number of nodes = {:15}\n"
3204 " Number of elements = {:15}\n"
3205 " Number of element blocks = {:15}\n"
3206 " Number of assemblies = {:15}\n\n"
3207 " Number of nodal point sets = {:15}\n"
3208 " Number of element side sets = {:15}\n\n"
3209 " Number of edge blocks = {:15}\n"
3210 " Number of face blocks = {:15}\n\n",
3211 global.title, fmt::group_digits(global.dimensionality),
3212 fmt::group_digits(global.nodeCount), fmt::group_digits(global.elementCount),
3213 fmt::group_digits(global.count(Excn::ObjectType::EBLK)),
3214 fmt::group_digits(global.count(Excn::ObjectType::ASSM)),
3215 fmt::group_digits(global.count(Excn::ObjectType::NSET)),
3216 fmt::group_digits(global.count(Excn::ObjectType::SSET)),
3217 fmt::group_digits(global.count(Excn::ObjectType::EDBLK)),
3218 fmt::group_digits(global.count(Excn::ObjectType::FABLK)));
3219 }
3220 }
3221
3222 template <typename T, typename INT>
3223 void get_nodesets(int part_count, size_t total_node_count,
3224 const std::vector<std::vector<INT>> &local_node_to_global,
3225 std::vector<std::vector<NodeSet<INT>>> &nodesets,
3226 std::vector<NodeSet<INT>> &glob_sets)
3227 {
3228 // Find number of nodesets in the global model...
3229 std::set<ex_entity_id> set_ids;
3230 ExodusIdVector ids;
3231
3232 int bad_ns = 0;
3233 for (int p = 0; p < part_count; p++) {
3234 ExodusFile id(p);
3235 int ns_count = ex_inquire_int(id, EX_INQ_NODE_SETS);
3236
3237 // Get the ids for these
3238 ids.resize(ns_count);
3239 int error = ex_get_ids(id, EX_NODE_SET, Data(ids));
3240 if (error < 0) {
3241 exodus_error(__LINE__);
3242 }
3243
3244 for (int iset = 0; iset < ns_count; iset++) {
3245 if (ids[iset] != 0) {
3246 set_ids.insert(ids[iset]);
3247 }
3248 else {
3249 bad_ns++;
3250 }
3251 }
3252 }
3253
3254 if (bad_ns != 0) {
3255 fmt::print(
3256 stderr,
3257 "ERROR: (EPU) There were {} nodesets (counting all files) which had an id equal to "
3258 "0 which is not allowed.\n",
3259 bad_ns);
3260 }
3261
3262 if (set_ids.empty()) {
3263 return;
3264 }
3265
3266 // set_ids now contains the union of all nodeset ids...
3267 for (int p = 0; p < part_count; p++) {
3268 ExodusFile id(p);
3269
3270 nodesets[p].resize(set_ids.size());
3271 auto I = set_ids.begin();
3272 auto IE = set_ids.end();
3273
3274 // Get the ids again so we can map current order back to file order...
3275 int error = ex_get_ids(id, EX_NODE_SET, Data(ids));
3276 if (error < 0) {
3277 exodus_error(__LINE__);
3278 }
3279
3280 std::vector<ex_set> sets(set_ids.size());
3281 int i = 0;
3282 while (I != IE) {
3283 sets[i].id = *I++;
3284 sets[i].type = EX_NODE_SET;
3285 sets[i].entry_list = nullptr;
3286 sets[i].extra_list = nullptr;
3287 sets[i].distribution_factor_list = nullptr;
3288 i++;
3289 }
3290
3291 error = ex_get_sets(id, set_ids.size(), Data(sets));
3292 if (error < 0) {
3293 exodus_error(__LINE__);
3294 }
3295
3296 for (size_t iset = 0; iset < set_ids.size(); iset++) {
3297 nodesets[p][iset].id = sets[iset].id;
3298 nodesets[p][iset].nodeCount = sets[iset].num_entry;
3299 nodesets[p][iset].dfCount = sets[iset].num_distribution_factor;
3300
3301 std::vector<char> name(Excn::ExodusFile::max_name_length() + 1);
3302 error = ex_get_name(id, EX_NODE_SET, nodesets[p][iset].id, Data(name));
3303 if (error < 0) {
3304 exodus_error(__LINE__);
3305 }
3306 if (name[0] != '\0') {
3307 nodesets[p][iset].name_ = Data(name);
3308 }
3309
3310 if (nodesets[p][iset].dfCount != 0 &&
3311 nodesets[p][iset].dfCount != nodesets[p][iset].nodeCount) {
3312 fmt::print(stderr,
3313 "WARNING: Nodeset {} with id {} on processor {} has an invalid distribution "
3314 "factor count ({})."
3315 " The distribution factors will be set to 1.0 on this nodeset.\n",
3316 nodesets[p][iset].name_, nodesets[p][iset].id, p,
3317 fmt::group_digits(nodesets[p][iset].dfCount));
3318 nodesets[p][iset].dfCount = 0;
3319 }
3320
3321 nodesets[p][iset].position_ = -1;
3322 for (size_t j = 0; j < ids.size(); j++) {
3323 if (nodesets[p][iset].id == ids[j]) {
3324 nodesets[p][iset].position_ = j;
3325 break;
3326 }
3327 }
3328
3329 if (debug_level & 32) {
3330 fmt::print("Processor {} ", p);
3331 nodesets[p][iset].dump();
3332 }
3333 }
3334 }
3335
3336 glob_sets.resize(set_ids.size());
3337 {
3338 // Now get the nodeset nodes and df.
3339
3340 // This is inefficient since the processor loop is on
3341 // the inside... The other ordering would use more memory...
3342
3343 std::vector<INT> ns_nodes;
3344 std::vector<T> ns_df;
3345
3346 for (size_t ns = 0; ns < set_ids.size(); ns++) {
3347
3348 std::vector<INT> glob_ns_nodes(total_node_count + 1);
3349 std::vector<T> glob_ns_df(total_node_count + 1);
3350
3351 for (int p = 0; p < part_count; p++) {
3352 ExodusFile id(p);
3353 if (p == 0) {
3354 glob_sets[ns].name_ = nodesets[p][ns].name_;
3355 glob_sets[ns].id = nodesets[p][ns].id;
3356 }
3357
3358 if (nodesets[p][ns].position_ != -1) {
3359 if (glob_sets[ns].position_ != -1) {
3360 SMART_ASSERT(glob_sets[ns].position_ == nodesets[p][ns].position_);
3361 }
3362 else {
3363 glob_sets[ns].position_ = nodesets[p][ns].position_;
3364 }
3365 }
3366
3367 size_t size = nodesets[p][ns].entity_count();
3368 if (size > 0) {
3369 ns_nodes.resize(size);
3370 ns_df.resize(size);
3371
3372 int error = ex_get_set(id, EX_NODE_SET, nodesets[p][ns].id, Data(ns_nodes), nullptr);
3373 if (error < 0) {
3374 exodus_error(__LINE__);
3375 }
3376 if (nodesets[p][ns].dfCount > 0) {
3377 ex_get_set_dist_fact(id, EX_NODE_SET, nodesets[p][ns].id, Data(ns_df));
3378 }
3379 else {
3380 std::fill(ns_df.begin(), ns_df.end(), T(1.0));
3381 }
3382
3383 // The node ids are in local space -- map to global; bring df along (if any).
3384 for (size_t iset = 0; iset < size; iset++) {
3385 SMART_ASSERT(ns_nodes[iset] > 0)(p)(ns)(iset)(ns_nodes[iset]);
3386 size_t global_node = local_node_to_global[p][ns_nodes[iset] - 1] + 1;
3387 glob_ns_nodes[global_node] = 1;
3388 glob_ns_df[global_node] = ns_df[iset];
3389 }
3390 }
3391 }
3392 // Count number of nonzero entries and transfer to the
3393 // output nodeset
3394 // NOTE: global_node above is 1-based.
3395 glob_sets[ns].nodeCount =
3396 std::accumulate(glob_ns_nodes.begin(), glob_ns_nodes.end(), (INT)0);
3397 glob_sets[ns].nodeSetNodes.resize(glob_sets[ns].entity_count());
3398 glob_sets[ns].dfCount = glob_sets[ns].nodeCount;
3399
3400 // distFactors is a vector of 'char' to allow storage of either float or double.
3401 glob_sets[ns].distFactors.resize(glob_sets[ns].dfCount * ExodusFile::io_word_size());
3402
3403 T *glob_df = (T *)(Data(glob_sets[ns].distFactors));
3404 size_t j = 0;
3405 for (size_t i = 1; i <= total_node_count; i++) {
3406 if (glob_ns_nodes[i] == 1) {
3407 glob_df[j] = glob_ns_df[i];
3408 glob_sets[ns].nodeSetNodes[j++] = i;
3409 glob_ns_nodes[i] = j;
3410 }
3411 }
3412
3413 SMART_ASSERT(j == glob_sets[ns].entity_count())(j)(glob_sets[ns].entity_count())(ns);
3414
3415 // See if we need a map from local nodeset position to global nodeset position
3416 // Only needed if there are nodeset variables
3417 // Assume all files have same number of variables...
3418 int num_vars;
3419 {
3420 ExodusFile id(0);
3421 int error = ex_get_variable_param(id, EX_NODE_SET, &num_vars);
3422 if (error < 0) {
3423 exodus_error(__LINE__);
3424 }
3425 }
3426 if (num_vars > 0) {
3427 for (int p = 0; p < part_count; p++) {
3428 ExodusFile id(p);
3429 // Get the nodelist, but store it in nodeOrderMap.
3430 // global_pos = nodeOrderMap[i]
3431 NodeSet<INT> &nset = nodesets[p][ns];
3432 size_t nnodes = nset.entity_count();
3433 nset.nodeOrderMap.resize(nnodes);
3434 int error = ex_get_set(id, EX_NODE_SET, nset.id, Data(nset.nodeOrderMap), nullptr);
3435 if (error < 0) {
3436 exodus_error(__LINE__);
3437 }
3438
3439 for (size_t i = 0; i < nnodes; i++) {
3440 size_t local_node = nset.nodeOrderMap[i]; // 1-based
3441 size_t global_node = local_node_to_global[p][local_node - 1] + 1; // 1-based
3442 size_t global_pos = glob_ns_nodes[global_node]; // 1-based
3443 nset.nodeOrderMap[i] = global_pos - 1;
3444 }
3445 #if 0
3446 if (debug_level & 32)
3447 nset.dump_order();
3448 #endif
3449 }
3450 }
3451 }
3452 }
3453 }
3454
3455 template <typename INT> void put_nodesets(std::vector<NodeSet<INT>> &glob_sets)
3456 {
3457 int exoid = ExodusFile::output();
3458
3459 if (debug_level & 32) {
3460 fmt::print("\nOutput NodeSets:\n");
3461 }
3462 for (auto &glob_set : glob_sets) {
3463 int error = ex_put_set(exoid, EX_NODE_SET, glob_set.id, Data(glob_set.nodeSetNodes), nullptr);
3464 if (error < 0) {
3465 exodus_error(__LINE__);
3466 }
3467 if (glob_set.dfCount > 0) {
3468 error = ex_put_set_dist_fact(exoid, EX_NODE_SET, glob_set.id, Data(glob_set.distFactors));
3469 if (error < 0) {
3470 exodus_error(__LINE__);
3471 }
3472 }
3473
3474 // Done with the memory; clear out the vector containing the bulk data nodes and distFactors.
3475 clear(glob_set.nodeSetNodes);
3476 clear(glob_set.distFactors);
3477
3478 if (debug_level & 32) {
3479 glob_set.dump();
3480 }
3481 }
3482 }
3483
3484 template <typename INT>
3485 void get_sideset_metadata(int part_count, std::vector<std::vector<SideSet<INT>>> &sets,
3486 std::vector<SideSet<INT>> &glob_ssets)
3487 {
3488 // Find number of sidesets in the global model...
3489 std::set<ex_entity_id> set_ids;
3490 ExodusIdVector ids;
3491
3492 int bad_ss = 0;
3493 {
3494 for (int p = 0; p < part_count; p++) {
3495 ExodusFile id(p);
3496 int ss_count = ex_inquire_int(id, EX_INQ_SIDE_SETS);
3497
3498 // Get the ids for these
3499 ids.resize(ss_count);
3500 int error = ex_get_ids(id, EX_SIDE_SET, Data(ids));
3501 if (error < 0) {
3502 exodus_error(__LINE__);
3503 }
3504 for (int i = 0; i < ss_count; i++) {
3505 if (ids[i] != 0) {
3506 set_ids.insert(ids[i]);
3507 }
3508 else {
3509 bad_ss++;
3510 }
3511 }
3512 }
3513 }
3514
3515 if (bad_ss != 0) {
3516 fmt::print(stderr,
3517 "ERROR: (EPU) There were {} sidesets (counting all files) which had an id equal "
3518 "to 0 which is not allowed.\n",
3519 bad_ss);
3520 }
3521
3522 if (set_ids.empty()) {
3523 return;
3524 }
3525
3526 // set_ids now contains the union of all sideset ids...
3527 glob_ssets.resize(set_ids.size());
3528
3529 {
3530 for (int p = 0; p < part_count; p++) {
3531 ExodusFile id(p);
3532
3533 sets[p].resize(set_ids.size());
3534 auto I = set_ids.begin();
3535 auto IE = set_ids.end();
3536
3537 // Get the ids again so we can map current order back to file order...
3538 int error = ex_get_ids(id, EX_SIDE_SET, Data(ids));
3539 if (error < 0) {
3540 exodus_error(__LINE__);
3541 }
3542
3543 std::vector<ex_set> exosets(set_ids.size());
3544 int j = 0;
3545 while (I != IE) {
3546 exosets[j].id = *I++;
3547 exosets[j].type = EX_SIDE_SET;
3548 exosets[j].entry_list = nullptr;
3549 exosets[j].extra_list = nullptr;
3550 exosets[j].distribution_factor_list = nullptr;
3551 j++;
3552 }
3553
3554 error = ex_get_sets(id, set_ids.size(), Data(exosets));
3555 if (error < 0) {
3556 exodus_error(__LINE__);
3557 }
3558
3559 for (size_t i = 0; i < set_ids.size(); i++) {
3560 sets[p][i].id = exosets[i].id;
3561 sets[p][i].sideCount = exosets[i].num_entry;
3562 sets[p][i].dfCount = exosets[i].num_distribution_factor;
3563
3564 glob_ssets[i].id = sets[p][i].id;
3565 glob_ssets[i].sideCount += sets[p][i].entity_count();
3566 glob_ssets[i].dfCount += sets[p][i].dfCount;
3567
3568 std::vector<char> name(Excn::ExodusFile::max_name_length() + 1);
3569 error = ex_get_name(id, EX_SIDE_SET, sets[p][i].id, Data(name));
3570 if (error < 0) {
3571 exodus_error(__LINE__);
3572 }
3573 if (name[0] != '\0') {
3574 sets[p][i].name_ = Data(name);
3575 if (p == 0) {
3576 glob_ssets[i].name_ = Data(name);
3577 }
3578 }
3579
3580 sets[p][i].position_ = -1;
3581 for (size_t jj = 0; jj < ids.size(); jj++) {
3582 if (sets[p][i].id == ids[jj]) {
3583 sets[p][i].position_ = jj;
3584 break;
3585 }
3586 }
3587
3588 if (sets[p][i].position_ != -1) {
3589 if (glob_ssets[i].position_ != -1) {
3590 SMART_ASSERT(glob_ssets[i].position_ == sets[p][i].position_);
3591 }
3592 else {
3593 glob_ssets[i].position_ = sets[p][i].position_;
3594 }
3595 }
3596 }
3597 }
3598
3599 // Calculate sideset offset
3600 for (size_t b = 0; b < glob_ssets.size(); b++) {
3601 size_t sum = 0;
3602 for (int p = 0; p < part_count; p++) {
3603 sets[p][b].offset_ = sum;
3604 sum += sets[p][b].entity_count();
3605
3606 if (debug_level & 16) {
3607 fmt::print("Processor {} ", p);
3608 sets[p][b].dump();
3609 }
3610 }
3611 }
3612 }
3613 }
3614
3615 template <typename INT>
3616 void get_put_sidesets(int part_count,
3617 const std::vector<std::vector<INT>> &local_element_to_global,
3618 std::vector<std::vector<SideSet<INT>>> &sets,
3619 std::vector<SideSet<INT>> &glob_ssets, Excn::SystemInterface &interFace)
3620 {
3621 // TODO(gdsjaar): See what work is really needed if in append mode...
3622
3623 // Get a temporary vector to maintain the current
3624 // offset into the glob_ssets for storing sides
3625 Int64Vector offset(glob_ssets.size());
3626 std::fill(offset.begin(), offset.end(), int64_t(0));
3627
3628 Int64Vector df_offset(glob_ssets.size());
3629 std::fill(df_offset.begin(), df_offset.end(), int64_t(0));
3630
3631 {
3632 for (auto &glob_sset : glob_ssets) {
3633 glob_sset.elems.resize(glob_sset.entity_count());
3634 glob_sset.sides.resize(glob_sset.entity_count());
3635 glob_sset.distFactors.resize(glob_sset.dfCount * ExodusFile::io_word_size());
3636 }
3637 }
3638
3639 // Now get the sideset elements, sides and df.
3640 for (int p = 0; p < part_count; p++) {
3641 ExodusFile id(p);
3642 for (size_t ss = 0; ss < glob_ssets.size(); ss++) {
3643
3644 size_t size = sets[p][ss].entity_count();
3645 if (size > 0) {
3646 size_t off = offset[ss];
3647 int error = ex_get_set(id, EX_SIDE_SET, sets[p][ss].id, &glob_ssets[ss].elems[off],
3648 &glob_ssets[ss].sides[off]);
3649 if (error < 0) {
3650 exodus_error(__LINE__);
3651 }
3652
3653 // The element ids are in local space -- map to global
3654 for (size_t i = 0; i < size; i++) {
3655 size_t local_elem = glob_ssets[ss].elems[off + i];
3656 SMART_ASSERT(local_elem > 0)(p)(ss)(i)(local_elem);
3657 SMART_ASSERT(glob_ssets[ss].sides[off + i] > 0 && glob_ssets[ss].sides[off + i] <= 6);
3658 size_t global_elem = local_element_to_global[p][local_elem - 1];
3659 glob_ssets[ss].elems[off + i] = global_elem + 1;
3660 }
3661 offset[ss] += size;
3662 }
3663
3664 // Distribution factors...
3665 size_t df_size = sets[p][ss].dfCount;
3666 if (df_size > 0) {
3667 size_t df_off = df_offset[ss] * ExodusFile::io_word_size();
3668 int error = ex_get_set_dist_fact(id, EX_SIDE_SET, sets[p][ss].id,
3669 &glob_ssets[ss].distFactors[df_off]);
3670 if (error < 0) {
3671 exodus_error(__LINE__);
3672 }
3673 df_offset[ss] += df_size;
3674 }
3675 }
3676 }
3677
3678 if (debug_level & 16) {
3679 fmt::print("\nOutput SideSets:\n");
3680 for (auto &glob_sset : glob_ssets) {
3681 glob_sset.dump();
3682 }
3683 }
3684
3685 if (!interFace.append()) {
3686 // Now write the actual sideset data...
3687 int exoid = ExodusFile::output(); // output file identifier
3688 for (auto &glob_sset : glob_ssets) {
3689 int error =
3690 ex_put_set(exoid, EX_SIDE_SET, glob_sset.id, const_cast<INT *>(Data(glob_sset.elems)),
3691 const_cast<INT *>(Data(glob_sset.sides)));
3692 if (error < 0) {
3693 exodus_error(__LINE__);
3694 }
3695 if (glob_sset.dfCount > 0) {
3696 error = ex_put_set_dist_fact(exoid, EX_SIDE_SET, glob_sset.id,
3697 reinterpret_cast<void *>(Data(glob_sset.distFactors)));
3698 if (error < 0) {
3699 exodus_error(__LINE__);
3700 }
3701 }
3702 }
3703 }
3704
3705 for (auto &glob_sset : glob_ssets) {
3706 clear(glob_sset.elems);
3707 clear(glob_sset.sides);
3708 clear(glob_sset.distFactors);
3709 }
3710 }
3711
3712 template <typename INT>
3713 void get_edgeblocks(int part_count, const std::vector<Mesh> &local_mesh, const Mesh &global,
3714 std::vector<std::vector<EdgeBlock<INT>>> &edgeblocks,
3715 std::vector<EdgeBlock<INT>> &glob_edgeblocks)
3716 {
3717 LOG("\n\n**** GET EDGE BLOCK INFORMATION (INCL. ELEMENT ATTRIBUTES) ****\n");
3718
3719 for (int ip = 0; ip < part_count; ip++) {
3720 edgeblocks[ip].resize(local_mesh[ip].count(Excn::ObjectType::EDBLK));
3721 }
3722
3723 if (rank == 0) {
3724 fmt::print("Global edge block count = {}\n",
3725 fmt::group_digits(global.count(Excn::ObjectType::EDBLK)));
3726 }
3727
3728 if (global.count(Excn::ObjectType::EDBLK) == 0) {
3729 return;
3730 }
3731
3732 ExodusIdVector edgeblock_id(global.count(Excn::ObjectType::EDBLK));
3733
3734 int error = 0;
3735 for (int p = 0; p < part_count; p++) {
3736 ExodusFile id(p);
3737
3738 error = ex_get_ids(id, EX_EDGE_BLOCK, Data(edgeblock_id));
3739 if (error < 0) {
3740 exodus_error(__LINE__);
3741 }
3742
3743 // Check that the block id ordering is consistent among files...
3744 if (p > 0) {
3745 for (size_t b = 0; b < global.count(Excn::ObjectType::EDBLK); b++) {
3746 if (edgeblocks[0][b].id != edgeblock_id[b]) {
3747 std::ostringstream errmsg;
3748 fmt::print(errmsg,
3749 "ERROR: (EPU) The internal edge block id ordering for part {}\n"
3750 " is not consistent with the ordering for part 0.\n",
3751 p);
3752 throw std::runtime_error(errmsg.str());
3753 }
3754 }
3755 }
3756
3757 if ((debug_level & 64) != 0U) {
3758 fmt::print("\nGetting edge block info for processor {}...\n", p);
3759 }
3760 else {
3761 if (p == 0) {
3762 LOG("\nGetting edge block info.\n");
3763 }
3764 }
3765
3766 for (size_t b = 0; b < global.count(Excn::ObjectType::EDBLK); b++) {
3767 if ((debug_level & 64) != 0U) {
3768 fmt::print("Edge Block {}, Id = {}", b, edgeblock_id[b]);
3769 }
3770
3771 ex_block temp_block{};
3772 temp_block.id = edgeblock_id[b];
3773 temp_block.type = EX_EDGE_BLOCK;
3774 error = ex_get_block_param(id, &temp_block);
3775 if (error < 0) {
3776 exodus_error(__LINE__);
3777 }
3778
3779 std::vector<char> name(Excn::ExodusFile::max_name_length() + 1);
3780 error = ex_get_name(id, EX_EDGE_BLOCK, edgeblock_id[b], Data(name));
3781 if (error < 0) {
3782 exodus_error(__LINE__);
3783 }
3784
3785 edgeblocks[p][b].id = edgeblock_id[b];
3786 if (name[0] != '\0') {
3787 edgeblocks[p][b].name_ = Data(name);
3788 }
3789 if (p == 0) {
3790 glob_edgeblocks[b].id = edgeblock_id[b];
3791 if (name[0] != '\0') {
3792 glob_edgeblocks[b].name_ = Data(name);
3793 }
3794 }
3795
3796 if (temp_block.num_entry != 0) {
3797 edgeblocks[p][b].edgeCount = temp_block.num_entry;
3798 edgeblocks[p][b].nodesPerEdge = temp_block.num_nodes_per_entry;
3799 edgeblocks[p][b].attributeCount = temp_block.num_attribute;
3800 edgeblocks[p][b].offset_ = temp_block.num_entry;
3801 edgeblocks[p][b].position_ = b;
3802 copy_string(edgeblocks[p][b].elType, temp_block.topology);
3803
3804 glob_edgeblocks[b].edgeCount += temp_block.num_entry;
3805 glob_edgeblocks[b].nodesPerEdge = temp_block.num_nodes_per_entry;
3806 glob_edgeblocks[b].attributeCount = temp_block.num_attribute;
3807 glob_edgeblocks[b].position_ = (int)b;
3808 copy_string(glob_edgeblocks[b].elType, temp_block.topology);
3809 }
3810
3811 if (temp_block.num_attribute > 0 && glob_edgeblocks[b].attributeNames.empty()) {
3812 // Get attribute names. Assume the same on all processors
3813 // on which the block exists.
3814 char **names = get_name_array(temp_block.num_attribute, ExodusFile::max_name_length());
3815
3816 error = ex_get_attr_names(id, EX_EDGE_BLOCK, edgeblock_id[b], names);
3817 if (error < 0) {
3818 exodus_error(__LINE__);
3819 }
3820 for (int i = 0; i < temp_block.num_attribute; i++) {
3821 glob_edgeblocks[b].attributeNames.emplace_back(names[i]);
3822 }
3823 free_name_array(names, temp_block.num_attribute);
3824 }
3825 if ((debug_level & 4) != 0U) {
3826 fmt::print(", Name = '{}', Edges = {:12}, Nodes/Edge= {}, Attributes = {}\n",
3827 edgeblocks[p][b].name_, fmt::group_digits(edgeblocks[p][b].entity_count()),
3828 edgeblocks[p][b].nodesPerEdge, edgeblocks[p][b].attributeCount);
3829 }
3830 }
3831 } // end for p=0..part_count
3832
3833 // Convert block_offset from edges/block/processor to true offset
3834 for (int p = 0; p < part_count; p++) {
3835 size_t sum = 0;
3836 for (size_t b = 0; b < global.count(Excn::ObjectType::EDBLK); b++) {
3837 size_t save = edgeblocks[p][b].offset_;
3838 edgeblocks[p][b].offset_ = sum;
3839 sum += save;
3840 }
3841 }
3842 }
3843
3844 template <typename T, typename INT>
3845 void put_edgeblocks(int part_count, int start_part,
3846 std::vector<std::vector<EdgeBlock<INT>>> &edgeblocks,
3847 std::vector<EdgeBlock<INT>> &glob_edgeblocks,
3848 const std::vector<std::vector<INT>> &local_node_to_global,
3849 const std::vector<std::vector<INT>> &local_edge_to_global)
3850 {
3851 SMART_ASSERT(sizeof(T) == ExodusFile::io_word_size());
3852 int global_num_edgeblocks = glob_edgeblocks.size();
3853
3854 auto linkage = new INT *[global_num_edgeblocks];
3855 auto attributes = new T *[global_num_edgeblocks];
3856
3857 LOG("\nReading and Writing edge connectivity & attributes\n");
3858
3859 for (int b = 0; b < global_num_edgeblocks; b++) {
3860
3861 if (debug_level & 64) {
3862 fmt::print("\nOutput edge block info for...\n"
3863 "Edge Block {}, Id = {}, Name = '{}', Edges = {:12}, Nodes/Edge = {}, "
3864 "Attributes = {}\n"
3865 "B{}:\t",
3866 b, glob_edgeblocks[b].id, glob_edgeblocks[b].name_,
3867 fmt::group_digits(glob_edgeblocks[b].entity_count()),
3868 glob_edgeblocks[b].nodesPerEdge, glob_edgeblocks[b].attributeCount, b);
3869 }
3870
3871 size_t max_nodes = glob_edgeblocks[b].entity_count();
3872 max_nodes *= glob_edgeblocks[b].nodesPerEdge;
3873
3874 if (max_nodes > 0) {
3875 linkage[b] = new INT[max_nodes];
3876 }
3877 else {
3878 linkage[b] = nullptr;
3879 }
3880 INT *block_linkage = linkage[b];
3881
3882 // Initialize attributes list, if it exists
3883 if (glob_edgeblocks[b].attributeCount > 0) {
3884 attributes[b] = new T[static_cast<size_t>(glob_edgeblocks[b].attributeCount) *
3885 glob_edgeblocks[b].entity_count()];
3886 }
3887
3888 int error = 0;
3889 for (int p = 0; p < part_count; p++) {
3890 ExodusFile id(p);
3891
3892 size_t global_pos;
3893 size_t global_block_pos;
3894
3895 if (edgeblocks[p][b].entity_count() > 0) { // non-zero length block
3896
3897 if (debug_level & 64) {
3898 fmt::print("#");
3899 }
3900 size_t maximum_nodes = edgeblocks[p][b].entity_count();
3901 maximum_nodes *= edgeblocks[p][b].nodesPerEdge;
3902 std::vector<INT> local_linkage(maximum_nodes);
3903 SMART_ASSERT(block_linkage != nullptr);
3904
3905 ex_entity_id bid = edgeblocks[p][b].id;
3906 error = ex_get_conn(id, EX_EDGE_BLOCK, bid, Data(local_linkage), nullptr, nullptr);
3907 if (error < 0) {
3908 fmt::print(stderr,
3909 "ERROR: (EPU) Cannot get edge block connectivity for block {} on part {}.\n",
3910 bid, p + start_part);
3911 exodus_error(__LINE__);
3912 }
3913 size_t pos = 0;
3914 size_t goffset = glob_edgeblocks[b].offset_;
3915 size_t edge_count = edgeblocks[p][b].entity_count();
3916 size_t boffset = edgeblocks[p][b].offset_;
3917 size_t npe = edgeblocks[p][b].nodesPerEdge;
3918 const std::vector<INT> &proc_loc_edge_to_global = local_edge_to_global[p];
3919 const std::vector<INT> &proc_loc_node_to_global = local_node_to_global[p];
3920
3921 for (size_t e = 0; e < edge_count; e++) {
3922 global_block_pos = proc_loc_edge_to_global[(e + boffset)] - goffset;
3923 global_pos = global_block_pos * npe;
3924
3925 for (size_t n = 0; n < npe; n++) {
3926 size_t node = proc_loc_node_to_global[local_linkage[pos++] - 1];
3927 block_linkage[global_pos++] = node + 1;
3928 }
3929 }
3930
3931 // Get attributes list, if it exists
3932 if (edgeblocks[p][b].attributeCount > 0) {
3933 size_t max_attr = edgeblocks[p][b].entity_count() * edgeblocks[p][b].attributeCount;
3934 std::vector<T> local_attr(max_attr);
3935
3936 error = ex_get_attr(id, EX_EDGE_BLOCK, edgeblocks[p][b].id, Data(local_attr));
3937 if (error < 0) {
3938 exodus_error(__LINE__);
3939 }
3940
3941 pos = 0;
3942
3943 size_t att_count = edgeblocks[p][b].attributeCount;
3944 for (size_t e = 0; e < edge_count; e++) {
3945 // global_pos is global position within this element block...
3946 global_block_pos = local_edge_to_global[p][(e + boffset)] - goffset;
3947 global_pos = global_block_pos * att_count;
3948 for (size_t n = 0; n < att_count; n++) {
3949 attributes[b][global_pos++] = local_attr[pos++];
3950 }
3951 }
3952 }
3953
3954 } // end if edgeblocks[p][b].entity_count() (non-zero length block)
3955 else if (debug_level & 64) {
3956 fmt::print(".");
3957 }
3958 } // end for p=0..part_count-1
3959
3960 // Write out block info
3961 int id_out = ExodusFile::output(); // output file identifier
3962
3963 if (linkage[b] != nullptr) {
3964 error =
3965 ex_put_conn(id_out, EX_EDGE_BLOCK, glob_edgeblocks[b].id, linkage[b], nullptr, nullptr);
3966 if (error < 0) {
3967 exodus_error(__LINE__);
3968 }
3969 delete[] linkage[b];
3970 }
3971
3972 // Write out attributes list if it exists
3973 if (glob_edgeblocks[b].attributeCount > 0) {
3974 error = ex_put_attr(id_out, EX_EDGE_BLOCK, glob_edgeblocks[b].id, attributes[b]);
3975 if (error < 0) {
3976 exodus_error(__LINE__);
3977 }
3978 delete[] attributes[b];
3979 } // end for b=0..global_num_edgeblocks-1
3980 if (debug_level & 64) {
3981 fmt::print("\n");
3982 }
3983 }
3984 fmt::print("\n");
3985 delete[] linkage;
3986 delete[] attributes;
3987 }
3988
3989 template <typename T, typename INT>
3990 void put_edgeblocks(int part_count, int start_part,
3991 std::vector<std::vector<EdgeBlock<INT>>> &edgeblocks,
3992 std::vector<EdgeBlock<INT>> &glob_edgeblocks,
3993 const std::vector<std::vector<INT>> &local_node_to_global)
3994 {
3995 // This variant of `put_edgeblocks` is used in the case of
3996 // `nomap` and uses much less memory (but may be slower). It
3997 // relies on the fact that with `nomap`, a the elements for a
3998 // block in a parts are all contiguous in the output file, so we
3999 // can read them, map the nodes, and then output them using a
4000 // partial write function and they don't need to be mapped into a
4001 // global array (which also does not need to be allocated). Phase
4002 // 1 of a "low-memory" option for epu.
4003
4004 SMART_ASSERT(sizeof(T) == ExodusFile::io_word_size());
4005 int global_num_edgeblocks = glob_edgeblocks.size();
4006
4007 LOG("\nReading and Writing edge connectivity & attributes (Low Memory Method)\n");
4008
4009 for (int b = 0; b < global_num_edgeblocks; b++) {
4010
4011 if (debug_level & 64) {
4012 fmt::print("\nOutput edge block info for...\n"
4013 "Edge Block {}, Id = {}, Name = '{}', Edges = {:12}, Nodes/Edge = {}, "
4014 "Attributes = {}\n",
4015 b, glob_edgeblocks[b].id, glob_edgeblocks[b].name_,
4016 fmt::group_digits(glob_edgeblocks[b].entity_count()),
4017 glob_edgeblocks[b].nodesPerEdge, glob_edgeblocks[b].attributeCount);
4018 }
4019
4020 int id_out = ExodusFile::output(); // output file identifier
4021
4022 size_t part_block_offset = 1;
4023 for (int p = 0; p < part_count; p++) {
4024 ExodusFile id(p);
4025
4026 if (edgeblocks[p][b].entity_count() > 0) { // non-zero length block
4027 size_t node_count = edgeblocks[p][b].entity_count() * edgeblocks[p][b].nodesPerEdge;
4028 std::vector<INT> local_linkage(node_count);
4029
4030 ex_entity_id bid = edgeblocks[p][b].id;
4031 int error = ex_get_conn(id, EX_EDGE_BLOCK, bid, Data(local_linkage), nullptr, nullptr);
4032 if (error < 0) {
4033 fmt::print(stderr,
4034 "ERROR: (EPU) Cannot get edge block connectivity for block {} on part {}.\n",
4035 bid, p + start_part);
4036 exodus_error(__LINE__);
4037 }
4038
4039 size_t edge_count = edgeblocks[p][b].entity_count();
4040 size_t npe = edgeblocks[p][b].nodesPerEdge;
4041 const std::vector<INT> &proc_loc_node_to_global = local_node_to_global[p];
4042
4043 size_t pos = 0;
4044 for (size_t e = 0; e < edge_count; e++) {
4045 for (size_t n = 0; n < npe; n++) {
4046 size_t node = proc_loc_node_to_global[local_linkage[pos] - 1];
4047 local_linkage[pos++] = node + 1;
4048 }
4049 }
4050
4051 if (debug_level & 64) {
4052 fmt::print(stderr, "part, block, offset, count = {} {} {} {}\n", p, bid,
4053 part_block_offset, fmt::group_digits(edge_count));
4054 }
4055 error = ex_put_partial_conn(id_out, EX_EDGE_BLOCK, bid, part_block_offset, edge_count,
4056 Data(local_linkage), nullptr, nullptr);
4057 if (error < 0) {
4058 fmt::print(stderr,
4059 "ERROR: (EPU) Cannot output edge block connectivity for block {} on part "
4060 "{} (offset = {}, count = {}).\n",
4061 bid, p + start_part, part_block_offset, fmt::group_digits(edge_count));
4062 exodus_error(__LINE__);
4063 }
4064
4065 // Get attributes list, if it exists
4066 if (edgeblocks[p][b].attributeCount > 0) {
4067 size_t max_attr = edgeblocks[p][b].entity_count() * edgeblocks[p][b].attributeCount;
4068 std::vector<T> local_attr(max_attr);
4069
4070 error = ex_get_attr(id, EX_EDGE_BLOCK, edgeblocks[p][b].id, Data(local_attr));
4071 if (error < 0) {
4072 exodus_error(__LINE__);
4073 }
4074
4075 error = ex_put_partial_attr(id_out, EX_EDGE_BLOCK, bid, part_block_offset, edge_count,
4076 Data(local_attr));
4077 if (error < 0) {
4078 exodus_error(__LINE__);
4079 }
4080 }
4081 part_block_offset += edge_count;
4082
4083 } // end if edgeblocks[p][b].entity_count() (non-zero length block)
4084 } // end for p=0..part_count-1
4085 }
4086 }
4087
4088 template <typename INT>
4089 void get_faceblocks(int part_count, const std::vector<Mesh> &local_mesh, const Mesh &global,
4090 std::vector<std::vector<FaceBlock<INT>>> &faceblocks,
4091 std::vector<FaceBlock<INT>> &glob_faceblocks)
4092 {
4093 LOG("\n\n**** GET FACE BLOCK INFORMATION (INCL. ELEMENT ATTRIBUTES) ****\n");
4094
4095 for (int ip = 0; ip < part_count; ip++) {
4096 faceblocks[ip].resize(local_mesh[ip].count(Excn::ObjectType::FABLK));
4097 }
4098
4099 if (rank == 0) {
4100 fmt::print("Global face block count = {}\n", global.count(Excn::ObjectType::FABLK));
4101 }
4102
4103 if (global.count(Excn::ObjectType::FABLK) == 0) {
4104 return;
4105 }
4106
4107 ExodusIdVector faceblock_id(global.count(Excn::ObjectType::FABLK));
4108
4109 int error = 0;
4110 for (int p = 0; p < part_count; p++) {
4111 ExodusFile id(p);
4112
4113 error = ex_get_ids(id, EX_FACE_BLOCK, Data(faceblock_id));
4114 if (error < 0) {
4115 exodus_error(__LINE__);
4116 }
4117
4118 // Check that the block id ordering is consistent among files...
4119 if (p > 0) {
4120 for (size_t b = 0; b < global.count(Excn::ObjectType::FABLK); b++) {
4121 if (faceblocks[0][b].id != faceblock_id[b]) {
4122 std::ostringstream errmsg;
4123 fmt::print(errmsg,
4124 "ERROR: (EPU) The internal face block id ordering for part {}\n"
4125 " is not consistent with the ordering for part 0.\n",
4126 p);
4127 throw std::runtime_error(errmsg.str());
4128 }
4129 }
4130 }
4131
4132 if ((debug_level & 64) != 0U) {
4133 fmt::print("\nGetting face block info for processor {}...\n", p);
4134 }
4135 else {
4136 if (p == 0) {
4137 LOG("\nGetting face block info.\n");
4138 }
4139 }
4140
4141 for (size_t b = 0; b < global.count(Excn::ObjectType::FABLK); b++) {
4142 if ((debug_level & 64) != 0U) {
4143 fmt::print("Face Block {}, Id = {}", b, faceblock_id[b]);
4144 }
4145
4146 ex_block temp_block{};
4147 temp_block.id = faceblock_id[b];
4148 temp_block.type = EX_FACE_BLOCK;
4149 error = ex_get_block_param(id, &temp_block);
4150 if (error < 0) {
4151 exodus_error(__LINE__);
4152 }
4153
4154 std::vector<char> name(Excn::ExodusFile::max_name_length() + 1);
4155 error = ex_get_name(id, EX_FACE_BLOCK, faceblock_id[b], Data(name));
4156 if (error < 0) {
4157 exodus_error(__LINE__);
4158 }
4159
4160 faceblocks[p][b].id = faceblock_id[b];
4161 if (name[0] != '\0') {
4162 faceblocks[p][b].name_ = Data(name);
4163 }
4164 if (p == 0) {
4165 glob_faceblocks[b].id = faceblock_id[b];
4166 if (name[0] != '\0') {
4167 glob_faceblocks[b].name_ = Data(name);
4168 }
4169 }
4170
4171 if (temp_block.num_entry != 0) {
4172 faceblocks[p][b].faceCount = temp_block.num_entry;
4173 faceblocks[p][b].nodesPerFace = temp_block.num_nodes_per_entry;
4174 faceblocks[p][b].attributeCount = temp_block.num_attribute;
4175 faceblocks[p][b].offset_ = temp_block.num_entry;
4176 faceblocks[p][b].position_ = b;
4177 copy_string(faceblocks[p][b].elType, temp_block.topology);
4178
4179 glob_faceblocks[b].faceCount += temp_block.num_entry;
4180 glob_faceblocks[b].nodesPerFace = temp_block.num_nodes_per_entry;
4181 glob_faceblocks[b].attributeCount = temp_block.num_attribute;
4182 glob_faceblocks[b].position_ = (int)b;
4183 copy_string(glob_faceblocks[b].elType, temp_block.topology);
4184 }
4185
4186 if (temp_block.num_attribute > 0 && glob_faceblocks[b].attributeNames.empty()) {
4187 // Get attribute names. Assume the same on all processors
4188 // on which the block exists.
4189 char **names = get_name_array(temp_block.num_attribute, ExodusFile::max_name_length());
4190
4191 error = ex_get_attr_names(id, EX_FACE_BLOCK, faceblock_id[b], names);
4192 if (error < 0) {
4193 exodus_error(__LINE__);
4194 }
4195 for (int i = 0; i < temp_block.num_attribute; i++) {
4196 glob_faceblocks[b].attributeNames.emplace_back(names[i]);
4197 }
4198 free_name_array(names, temp_block.num_attribute);
4199 }
4200 if ((debug_level & 4) != 0U) {
4201 fmt::print(", Name = '{}', Faces = {:12}, Nodes/Face= {}, Attributes = {}\n",
4202 faceblocks[p][b].name_, fmt::group_digits(faceblocks[p][b].entity_count()),
4203 faceblocks[p][b].nodesPerFace, faceblocks[p][b].attributeCount);
4204 }
4205 }
4206 } // end for p=0..part_count
4207
4208 // Convert block_offset from faces/block/processor to true offset
4209 for (int p = 0; p < part_count; p++) {
4210 size_t sum = 0;
4211 for (size_t b = 0; b < global.count(Excn::ObjectType::FABLK); b++) {
4212 size_t save = faceblocks[p][b].offset_;
4213 faceblocks[p][b].offset_ = sum;
4214 sum += save;
4215 }
4216 }
4217 }
4218
4219 template <typename T, typename INT>
4220 void put_faceblocks(int part_count, int start_part,
4221 std::vector<std::vector<FaceBlock<INT>>> &faceblocks,
4222 std::vector<FaceBlock<INT>> &glob_faceblocks,
4223 const std::vector<std::vector<INT>> &local_node_to_global,
4224 const std::vector<std::vector<INT>> &local_face_to_global)
4225 {
4226 SMART_ASSERT(sizeof(T) == ExodusFile::io_word_size());
4227 int global_num_faceblocks = glob_faceblocks.size();
4228
4229 auto linkage = new INT *[global_num_faceblocks];
4230 auto attributes = new T *[global_num_faceblocks];
4231
4232 LOG("\nReading and Writing face connectivity & attributes\n");
4233
4234 for (int b = 0; b < global_num_faceblocks; b++) {
4235
4236 if (debug_level & 64) {
4237 fmt::print("\nOutput face block info for...\n"
4238 "Face Block {}, Id = {}, Name = '{}', Faces = {:12}, Nodes/Face = {}, "
4239 "Attributes = {}\n"
4240 "B{}:\t",
4241 b, glob_faceblocks[b].id, glob_faceblocks[b].name_,
4242 fmt::group_digits(glob_faceblocks[b].entity_count()),
4243 glob_faceblocks[b].nodesPerFace, glob_faceblocks[b].attributeCount, b);
4244 }
4245
4246 size_t max_nodes = glob_faceblocks[b].entity_count();
4247 max_nodes *= glob_faceblocks[b].nodesPerFace;
4248
4249 if (max_nodes > 0) {
4250 linkage[b] = new INT[max_nodes];
4251 }
4252 else {
4253 linkage[b] = nullptr;
4254 }
4255 INT *block_linkage = linkage[b];
4256
4257 // Initialize attributes list, if it exists
4258 if (glob_faceblocks[b].attributeCount > 0) {
4259 attributes[b] = new T[static_cast<size_t>(glob_faceblocks[b].attributeCount) *
4260 glob_faceblocks[b].entity_count()];
4261 }
4262
4263 int error = 0;
4264 for (int p = 0; p < part_count; p++) {
4265 ExodusFile id(p);
4266
4267 size_t global_pos;
4268 size_t global_block_pos;
4269
4270 if (faceblocks[p][b].entity_count() > 0) { // non-zero length block
4271
4272 if (debug_level & 64) {
4273 fmt::print("#");
4274 }
4275 size_t maximum_nodes = faceblocks[p][b].entity_count();
4276 maximum_nodes *= faceblocks[p][b].nodesPerFace;
4277 std::vector<INT> local_linkage(maximum_nodes);
4278 SMART_ASSERT(block_linkage != nullptr);
4279
4280 ex_entity_id bid = faceblocks[p][b].id;
4281 error = ex_get_conn(id, EX_FACE_BLOCK, bid, Data(local_linkage), nullptr, nullptr);
4282 if (error < 0) {
4283 fmt::print(stderr,
4284 "ERROR: (EPU) Cannot get face block connectivity for block {} on part {}.\n",
4285 bid, p + start_part);
4286 exodus_error(__LINE__);
4287 }
4288 size_t pos = 0;
4289 size_t goffset = glob_faceblocks[b].offset_;
4290 size_t face_count = faceblocks[p][b].entity_count();
4291 size_t boffset = faceblocks[p][b].offset_;
4292 size_t npe = faceblocks[p][b].nodesPerFace;
4293 const auto &proc_loc_face_to_global = local_face_to_global[p];
4294 const auto &proc_loc_node_to_global = local_node_to_global[p];
4295
4296 for (size_t e = 0; e < face_count; e++) {
4297 global_block_pos = proc_loc_face_to_global[(e + boffset)] - goffset;
4298 global_pos = global_block_pos * npe;
4299
4300 for (size_t n = 0; n < npe; n++) {
4301 size_t node = proc_loc_node_to_global[local_linkage[pos++] - 1];
4302 block_linkage[global_pos++] = node + 1;
4303 }
4304 }
4305
4306 // Get attributes list, if it exists
4307 if (faceblocks[p][b].attributeCount > 0) {
4308 size_t max_attr = faceblocks[p][b].entity_count() * faceblocks[p][b].attributeCount;
4309 std::vector<T> local_attr(max_attr);
4310
4311 error = ex_get_attr(id, EX_FACE_BLOCK, faceblocks[p][b].id, Data(local_attr));
4312 if (error < 0) {
4313 exodus_error(__LINE__);
4314 }
4315
4316 pos = 0;
4317
4318 size_t att_count = faceblocks[p][b].attributeCount;
4319 for (size_t e = 0; e < face_count; e++) {
4320 // global_pos is global position within this element block...
4321 global_block_pos = local_face_to_global[p][(e + boffset)] - goffset;
4322 global_pos = global_block_pos * att_count;
4323 for (size_t n = 0; n < att_count; n++) {
4324 attributes[b][global_pos++] = local_attr[pos++];
4325 }
4326 }
4327 }
4328
4329 } // end if faceblocks[p][b].entity_count() (non-zero length block)
4330 else if (debug_level & 64) {
4331 fmt::print(".");
4332 }
4333 } // end for p=0..part_count-1
4334
4335 // Write out block info
4336 int id_out = ExodusFile::output(); // output file identifier
4337
4338 if (linkage[b] != nullptr) {
4339 fmt::print(stderr, "(EPU) id_out({}) glob_faceblocks[{}].id({})\n", id_out, b,
4340 glob_faceblocks[b].id);
4341 error =
4342 ex_put_conn(id_out, EX_FACE_BLOCK, glob_faceblocks[b].id, linkage[b], nullptr, nullptr);
4343 if (error < 0) {
4344 exodus_error(__LINE__);
4345 }
4346 delete[] linkage[b];
4347 }
4348
4349 // Write out attributes list if it exists
4350 if (glob_faceblocks[b].attributeCount > 0) {
4351 error = ex_put_attr(id_out, EX_FACE_BLOCK, glob_faceblocks[b].id, attributes[b]);
4352 if (error < 0) {
4353 exodus_error(__LINE__);
4354 }
4355 delete[] attributes[b];
4356 } // end for b=0..global_num_faceblocks-1
4357 if (debug_level & 64) {
4358 fmt::print("\n");
4359 }
4360 }
4361 fmt::print("\n");
4362 delete[] linkage;
4363 delete[] attributes;
4364 }
4365
4366 template <typename T, typename INT>
4367 void put_faceblocks(int part_count, int start_part,
4368 std::vector<std::vector<FaceBlock<INT>>> &faceblocks,
4369 std::vector<FaceBlock<INT>> &glob_faceblocks,
4370 const std::vector<std::vector<INT>> &local_node_to_global)
4371 {
4372 // This variant of `put_faceblocks` is used in the case of
4373 // `nomap` and uses much less memory (but may be slower). It
4374 // relies on the fact that with `nomap`, a the elements for a
4375 // block in a parts are all contiguous in the output file, so we
4376 // can read them, map the nodes, and then output them using a
4377 // partial write function and they don't need to be mapped into a
4378 // global array (which also does not need to be allocated). Phase
4379 // 1 of a "low-memory" option for epu.
4380
4381 SMART_ASSERT(sizeof(T) == ExodusFile::io_word_size());
4382 int global_num_faceblocks = glob_faceblocks.size();
4383
4384 LOG("\nReading and Writing face connectivity & attributes (Low Memory Method)\n");
4385
4386 for (int b = 0; b < global_num_faceblocks; b++) {
4387
4388 if (debug_level & 64) {
4389 fmt::print("\nOutput face block info for...\n"
4390 "Face Block {}, Id = {}, Name = '{}', Faces = {:12}, Nodes/Face = {}, "
4391 "Attributes = {}\n",
4392 b, glob_faceblocks[b].id, glob_faceblocks[b].name_,
4393 fmt::group_digits(glob_faceblocks[b].entity_count()),
4394 glob_faceblocks[b].nodesPerFace, glob_faceblocks[b].attributeCount);
4395 }
4396
4397 int id_out = ExodusFile::output(); // output file identifier
4398
4399 size_t part_block_offset = 1;
4400 for (int p = 0; p < part_count; p++) {
4401 ExodusFile id(p);
4402
4403 if (faceblocks[p][b].entity_count() > 0) { // non-zero length block
4404 size_t node_count = faceblocks[p][b].entity_count() * faceblocks[p][b].nodesPerFace;
4405 std::vector<INT> local_linkage(node_count);
4406
4407 ex_entity_id bid = faceblocks[p][b].id;
4408 int error = ex_get_conn(id, EX_FACE_BLOCK, bid, Data(local_linkage), nullptr, nullptr);
4409 if (error < 0) {
4410 fmt::print(stderr,
4411 "ERROR: (EPU) Cannot get face block connectivity for block {} on part {}.\n",
4412 bid, p + start_part);
4413 exodus_error(__LINE__);
4414 }
4415
4416 size_t face_count = faceblocks[p][b].entity_count();
4417 size_t npe = faceblocks[p][b].nodesPerFace;
4418 const std::vector<INT> &proc_loc_node_to_global = local_node_to_global[p];
4419
4420 size_t pos = 0;
4421 for (size_t e = 0; e < face_count; e++) {
4422 for (size_t n = 0; n < npe; n++) {
4423 size_t node = proc_loc_node_to_global[local_linkage[pos] - 1];
4424 local_linkage[pos++] = node + 1;
4425 }
4426 }
4427
4428 if (debug_level & 64) {
4429 fmt::print(stderr, "part, block, offset, count = {} {} {} {}\n", p, bid,
4430 part_block_offset, face_count);
4431 }
4432 error = ex_put_partial_conn(id_out, EX_FACE_BLOCK, bid, part_block_offset, face_count,
4433 Data(local_linkage), nullptr, nullptr);
4434 if (error < 0) {
4435 fmt::print(stderr,
4436 "ERROR: (EPU) Cannot output face block connectivity for block {} on part "
4437 "{} (offset = {}, count = {}).\n",
4438 bid, p + start_part, part_block_offset, face_count);
4439 exodus_error(__LINE__);
4440 }
4441
4442 // Get attributes list, if it exists
4443 if (faceblocks[p][b].attributeCount > 0) {
4444 size_t max_attr = faceblocks[p][b].entity_count() * faceblocks[p][b].attributeCount;
4445 std::vector<T> local_attr(max_attr);
4446
4447 error = ex_get_attr(id, EX_FACE_BLOCK, faceblocks[p][b].id, Data(local_attr));
4448 if (error < 0) {
4449 exodus_error(__LINE__);
4450 }
4451
4452 error = ex_put_partial_attr(id_out, EX_FACE_BLOCK, bid, part_block_offset, face_count,
4453 Data(local_attr));
4454 if (error < 0) {
4455 exodus_error(__LINE__);
4456 }
4457 }
4458 part_block_offset += face_count;
4459
4460 } // end if faceblocks[p][b].entity_count() (non-zero length block)
4461 } // end for p=0..part_count-1
4462 }
4463 }
4464
4465 template <typename T, typename INT>
4466 void add_processor_variable(int id_out, int part_count, int start_part, const Mesh &global,
4467 std::vector<std::vector<Block>> &blocks,
4468 const std::vector<Block> &glob_blocks,
4469 const std::vector<std::vector<INT>> &local_element_to_global,
4470 int step, int variable, std::vector<T> &proc)
4471 {
4472 SMART_ASSERT(sizeof(T) == ExodusFile::io_word_size());
4473 for (size_t b = 0; b < global.count(Excn::ObjectType::EBLK); b++) {
4474 proc.resize(glob_blocks[b].entity_count());
4475 for (int p = 0; p < part_count; p++) {
4476 size_t boffset = blocks[p][b].offset_;
4477 size_t goffset = glob_blocks[b].offset_;
4478 size_t element_count = blocks[p][b].entity_count();
4479 for (size_t e = 0; e < element_count; e++) {
4480 size_t global_block_pos = local_element_to_global[p][(e + boffset)] - goffset;
4481 proc[global_block_pos] = p + start_part;
4482 }
4483 }
4484 int error = ex_put_var(id_out, step, EX_ELEM_BLOCK, variable, glob_blocks[b].id,
4485 glob_blocks[b].entity_count(), Data(proc));
4486 if (error < 0) {
4487 exodus_error(__LINE__);
4488 }
4489 }
4490 }
4491
4492 template <typename INT>
4493 void add_processor_map(int id_out, int part_count, int start_part, const Mesh &global,
4494 std::vector<std::vector<Block>> &blocks,
4495 const std::vector<Block> &glob_blocks,
4496 const std::vector<std::vector<INT>> &local_element_to_global)
4497 {
4498 std::vector<INT> proc(global.elementCount);
4499
4500 for (size_t b = 0; b < global.count(Excn::ObjectType::EBLK); b++) {
4501 proc.resize(glob_blocks[b].entity_count());
4502 for (int p = 0; p < part_count; p++) {
4503 size_t boffset = blocks[p][b].offset_;
4504 size_t element_count = blocks[p][b].entity_count();
4505 for (size_t e = 0; e < element_count; e++) {
4506 size_t global_elem_pos = local_element_to_global[p][(e + boffset)];
4507 proc[global_elem_pos] = p + start_part;
4508 }
4509 }
4510 }
4511
4512 if (ex_put_map_param(id_out, 0, 1) < 0) {
4513 exodus_error(__LINE__);
4514 }
4515 if (ex_put_num_map(id_out, EX_ELEM_MAP, 1, Data(proc)) < 0) {
4516 exodus_error(__LINE__);
4517 }
4518 if (ex_put_name(id_out, EX_ELEM_MAP, 1, "processor_id") < 0) {
4519 exodus_error(__LINE__);
4520 }
4521 }
4522
4523 StringVector get_exodus_variable_names(int id, ex_entity_type elType, int var_count)
4524 {
4525 // Allocate space for variable names...
4526 char **name_list = get_name_array(var_count, ExodusFile::max_name_length());
4527 int error = ex_get_variable_names(id, elType, var_count, name_list);
4528 if (error < 0) {
4529 exodus_error(__LINE__);
4530 }
4531
4532 StringVector names(var_count);
4533 for (int j = 0; j < var_count; j++) {
4534 compress_white_space(name_list[j]);
4535 names[j] = std::string(name_list[j]);
4536 }
4537
4538 free_name_array(name_list, var_count);
4539 return names;
4540 }
4541
4542 template <typename T>
4543 void filter_truth_table(int id, Mesh &global, std::vector<T> &glob_blocks, Variables &vars,
4544 const StringIdVector &variable_names)
4545 {
4546 // This routine checks the 'variable_names' list to see if the user
4547 // has restricted the output of certain variables to certain element
4548 // blocks. If so, then the truth table is modified to match the
4549 // users request.
4550 if (variable_names.empty()) {
4551 return;
4552 }
4553
4554 // Check for a non-zero id entry in the variable_names list which
4555 // will signify a user-specified element block.
4556 bool found_it = false;
4557 for (size_t i = 0; i < variable_names.size() && !found_it; i++) {
4558 if (variable_names[i].second > 0) {
4559 found_it = true;
4560 }
4561 }
4562 if (!found_it) {
4563 return;
4564 }
4565
4566 // At this point, know that there is at least one block-restricted
4567 // variable output specification. For each one, modify the global
4568 // truth table to match the specification.
4569 StringVector exo_names = get_exodus_variable_names(id, vars.type(), vars.count());
4570
4571 std::string var_name;
4572 int out_position = -1;
4573 for (const auto &[variable_name, variable_block] : variable_names) {
4574 if (variable_block > 0) {
4575 if (var_name != variable_name) {
4576 var_name = variable_name;
4577 // Find which exodus variable matches this name
4578 out_position = -1;
4579 for (size_t j = 0; j < exo_names.size(); j++) {
4580 if (case_compare(exo_names[j], var_name)) {
4581 out_position = vars.index_[j] - 1;
4582 break;
4583 }
4584 }
4585 SMART_ASSERT(out_position >= 0);
4586
4587 // Set all truth table entries for this variable to negative
4588 // of current value and then iterate over specified blocks and
4589 // set those positive. This way can make sure that the
4590 // variable truly exists for the block that the user specified.
4591 for (size_t b = 0; b < global.count(vars.objectType); b++) {
4592 int truth_table_loc = (b * vars.count(InOut::OUT)) + out_position;
4593 global.truthTable[static_cast<int>(vars.objectType)][truth_table_loc] *= -1;
4594 }
4595 }
4596 // Find out which block corresponds to the specified id.
4597 int block = -1;
4598 for (size_t b = 0; b < global.count(vars.objectType); b++) {
4599 if (glob_blocks[b].id == variable_block) {
4600 block = b;
4601 break;
4602 }
4603 }
4604
4605 if (block == -1) {
4606 std::ostringstream errmsg;
4607 fmt::print(
4608 errmsg,
4609 "ERROR: (EPU) User-specified block id of {} for variable '{}' does not exist.\n",
4610 variable_block, variable_name);
4611 throw std::runtime_error(errmsg.str());
4612 }
4613
4614 int truth_table_loc = block * vars.count(InOut::OUT) + out_position;
4615 if (global.truthTable[static_cast<int>(vars.objectType)][truth_table_loc] == 0) {
4616 std::ostringstream errmsg;
4617 fmt::print(errmsg, "ERROR: (EPU) Variable '{}' does not exist on block {}.\n",
4618 variable_name, variable_block);
4619 throw std::runtime_error(errmsg.str());
4620 }
4621 global.truthTable[static_cast<int>(vars.objectType)][truth_table_loc] = 1;
4622 }
4623 }
4624
4625 // reset truth table values that may be negative
4626 int output_truth_table_length = vars.count(InOut::OUT) * global.count(vars.objectType);
4627 for (int j = 0; j < output_truth_table_length; j++) {
4628 if (global.truthTable[static_cast<int>(vars.objectType)][j] < 0) {
4629 global.truthTable[static_cast<int>(vars.objectType)][j] = 0;
4630 }
4631 }
4632 }
4633
4634 template <typename T>
4635 void get_truth_table(Mesh &global, std::vector<T> &glob_blocks, std::vector<Mesh> &local,
4636 Variables &vars, int debug)
4637 {
4638 // read truth table - sum across all processors since many will
4639 // have values set to zero for zero length blocks the element
4640 // variable truth table organized as a 2D array:
4641 // [global.count(EBLK)][num_elem_vars]
4642
4643 ObjectType object_type = vars.objectType;
4644 int input_truth_table_length = vars.count(InOut::IN) * global.count(object_type);
4645 int output_truth_table_length = vars.count(InOut::OUT) * global.count(object_type);
4646
4647 if (output_truth_table_length) {
4648
4649 global.truthTable[static_cast<int>(object_type)].resize(output_truth_table_length);
4650 std::fill(global.truthTable[static_cast<int>(object_type)].begin(),
4651 global.truthTable[static_cast<int>(object_type)].end(), 0);
4652
4653 // For each input exodus file, get it's truth table and fill
4654 // in the location in the output truth table...
4655
4656 bool is_sidenodeset =
4657 vars.objectType == Excn::ObjectType::NSET || vars.objectType == Excn::ObjectType::SSET;
4658 int part_count = local.size();
4659 for (int p = 0; p < part_count; p++) {
4660 ExodusFile id(p);
4661
4662 if (vars.count(InOut::IN) > 0) { // Could be zero if add_processor_id
4663 // is the only variable...
4664 local[p].truthTable[static_cast<int>(object_type)].resize(input_truth_table_length);
4665 int error =
4666 ex_get_truth_table(id, vars.type(), global.count(object_type), vars.count(InOut::IN),
4667 Data(local[p].truthTable[static_cast<int>(object_type)]));
4668 if (error < 0) {
4669 exodus_error(__LINE__);
4670 }
4671 }
4672 for (size_t b = 0; b < global.count(object_type); b++) {
4673 size_t bin = b;
4674 if (is_sidenodeset) {
4675 bin = glob_blocks[b].position_;
4676 }
4677
4678 for (int j = 0; j < vars.count(InOut::IN); j++) {
4679 if (vars.index_[j] > 0) {
4680 int ki = (bin * vars.count(InOut::IN)) + j;
4681 int ko = (b * vars.count(InOut::OUT)) + vars.index_[j] - 1;
4682 SMART_ASSERT(ko < output_truth_table_length);
4683 SMART_ASSERT(ki < input_truth_table_length);
4684 global.truthTable[static_cast<int>(object_type)][ko] +=
4685 local[p].truthTable[static_cast<int>(object_type)][ki];
4686 }
4687 }
4688 if (vars.addProcessorId) {
4689 int ko = (b * vars.count(InOut::OUT)) + vars.count(InOut::OUT) - 1;
4690 SMART_ASSERT(ko < output_truth_table_length);
4691 global.truthTable[static_cast<int>(object_type)][ko] = 1;
4692 }
4693 }
4694 }
4695
4696 // reset truth table values that may be greater than 1
4697 for (int j = 0; j < output_truth_table_length; j++) {
4698 SMART_ASSERT(global.truthTable[static_cast<int>(object_type)][j] >= 0)
4699 (global.truthTable[static_cast<int>(object_type)][j]);
4700 if (global.truthTable[static_cast<int>(object_type)][j] > 0) {
4701 global.truthTable[static_cast<int>(object_type)][j] = 1;
4702 }
4703 }
4704
4705 if (debug_level & debug) {
4706 fmt::print("Truth table for {}\n", vars.label());
4707 int k = 0;
4708 for (size_t b = 0; b < global.count(object_type); b++) {
4709 for (int j = 0; j < vars.count(InOut::OUT); j++) {
4710 fmt::print("{}", global.truthTable[static_cast<int>(object_type)][k++]);
4711 }
4712 fmt::print("\n");
4713 }
4714 }
4715 }
4716 }
4717
4718 bool case_compare(const std::string &s1, const std::string &s2)
4719 {
4720 return (s1.size() == s2.size()) &&
4721 std::equal(s1.begin(), s1.end(), s2.begin(),
4722 [](char a, char b) { return std::tolower(a) == std::tolower(b); });
4723 }
4724
4725 void add_info_record(char *info_record, int size)
4726 {
4727 std::string info = sys_info("EPU");
4728 copy_string(info_record, info, size + 1);
4729 }
4730
4731 inline bool is_whitespace(char c)
4732 {
4733 static char white_space[] = {' ', '\t', '\n', '\r', ',', '\0'};
4734 return strchr(white_space, c) != nullptr;
4735 }
4736
4737 void compress_white_space(char *str)
4738 {
4739 char *ibuf = str;
4740 char *obuf = str;
4741
4742 int i = 0;
4743 int cnt = 0;
4744
4745 // Don't process an empty string.
4746 if (str == nullptr) {
4747 return;
4748 }
4749
4750 // Skip leading...
4751 while (*ibuf != 0 && is_whitespace(*ibuf)) {
4752 ++ibuf;
4753 }
4754
4755 while (*ibuf != 0) {
4756 if (is_whitespace(*ibuf) && cnt > 0) {
4757 ibuf++;
4758 }
4759 else {
4760 if (!is_whitespace(*ibuf)) {
4761 cnt = 0;
4762 }
4763 else {
4764 *ibuf = ' ';
4765 cnt = 1;
4766 }
4767 obuf[i++] = *ibuf++;
4768 }
4769 }
4770 obuf[i--] = '\0';
4771
4772 // Skip trailing whitespace
4773 while (i > 0 && is_whitespace(obuf[i])) {
4774 obuf[i--] = '\0';
4775 }
4776 }
4777
4778 int get_width(int max_value)
4779 {
4780 // Returns the field width which will accommodate the
4781 // largest value.
4782 int width = 1;
4783 if (max_value >= 10) {
4784 width = int(std::log10(static_cast<double>(max_value)));
4785 }
4786 return width + 1;
4787 }
4788
4789 template <typename T, typename INT>
4790 void map_element_vars(size_t loffset, size_t goffset, size_t entity_count, std::vector<T> &values,
4791 T *global_values, const std::vector<INT> &proc_loc_elem_to_global)
4792 {
4793 // copy values to master element value information
4794 T *local_values = Data(values);
4795 for (size_t j = 0; j < entity_count; j++) {
4796 size_t global_block_pos = proc_loc_elem_to_global[(j + loffset)] - goffset;
4797 global_values[global_block_pos] = local_values[j];
4798 }
4799 }
4800
4801 template <typename T>
4802 void map_sideset_vars(size_t loffset, size_t entity_count, std::vector<T> &values,
4803 T *global_values)
4804 {
4805 // copy values to master sideset value information
4806 T *local_values = Data(values);
4807 for (size_t j = 0; j < entity_count; j++) {
4808 global_values[j + loffset] = local_values[j];
4809 }
4810 }
4811
4812 template <typename T, typename U>
4813 void map_nodeset_vars(U & /*unused*/, size_t /*unused*/, size_t /*unused*/,
4814 std::vector<T> & /*unused*/, T * /*unused*/)
4815 {
4816 throw std::runtime_error("Internal Error!");
4817 }
4818
4819 template <typename INT>
4820 void map_nodeset_vars(Excn::NodeSet<INT> &local_set, size_t entity_count,
4821 size_t glob_entity_count, std::vector<double> &values,
4822 double *global_values)
4823 {
4824 // copy values to master nodeset value information
4825 double *local_values = Data(values);
4826 for (size_t j = 0; j < entity_count; j++) {
4827 size_t global_loc = local_set.nodeOrderMap[j];
4828 SMART_ASSERT(global_loc < glob_entity_count);
4829 global_values[global_loc] = local_values[j];
4830 }
4831 }
4832
4833 template <typename INT>
4834 void map_nodeset_vars(Excn::NodeSet<INT> &local_set, size_t entity_count,
4835 size_t glob_entity_count, std::vector<float> &values, float *global_values)
4836 {
4837 // copy values to master nodeset value information
4838 float *local_values = Data(values);
4839 for (size_t j = 0; j < entity_count; j++) {
4840 size_t global_loc = local_set.nodeOrderMap[j];
4841 SMART_ASSERT(global_loc < glob_entity_count);
4842 global_values[global_loc] = local_values[j];
4843 }
4844 }
4845
4846 template <typename T, typename INT>
4847 void map_edgeblock_vars(size_t loffset, size_t goffset, size_t entity_count,
4848 std::vector<T> &values, T *global_values,
4849 const std::vector<INT> &proc_loc_elem_to_global)
4850 {
4851 // copy values to master edgeblock value information
4852 T *local_values = Data(values);
4853 for (size_t j = 0; j < entity_count; j++) {
4854 size_t global_block_pos = proc_loc_elem_to_global[(j + loffset)] - goffset;
4855 global_values[global_block_pos] = local_values[j];
4856 }
4857 }
4858
4859 template <typename T, typename U, typename INT>
4860 void read_write_master_values(Excn::Variables &vars, const Excn::Mesh &global,
4861 std::vector<U> &global_sets, std::vector<Excn::Mesh> &local_mesh,
4862 std::vector<std::vector<U>> &local_sets,
4863 std::vector<T> &master_values, std::vector<T> &values,
4864 int part_count, int time_step, int time_step_out,
4865 const std::vector<std::vector<INT>> &local_entity_to_global)
4866 {
4867 bool is_sidenodeset =
4868 vars.objectType == Excn::ObjectType::NSET || vars.objectType == Excn::ObjectType::SSET;
4869
4870 int id_out = ExodusFile::output(); // output file identifier
4871
4872 for (int i = 0; i < vars.count(InOut::IN); i++) {
4873 if (vars.index_[i] > 0) {
4874 int ivar = vars.index_[i] - 1;
4875
4876 for (size_t b = 0; b < global.count(vars.objectType); b++) {
4877 size_t bin = b;
4878 if (is_sidenodeset) {
4879 bin = global_sets[b].position_;
4880 }
4881 int output_truth_table_loc = (b * vars.count(InOut::OUT)) + ivar;
4882 int input_truth_table_loc = (bin * vars.count(InOut::IN)) + i;
4883 if (debug_level & 4) {
4884 std::fill(master_values.begin(), master_values.end(), T(0.0));
4885 }
4886
4887 for (int p = 0; p < part_count; p++) {
4888 ExodusFile id(p);
4889
4890 const std::vector<INT> &proc_loc_elem_to_global = local_entity_to_global[p];
4891 const std::vector<INT> &proc_loc_edge_to_global = local_entity_to_global[p];
4892 const std::vector<INT> &proc_loc_face_to_global = local_entity_to_global[p];
4893
4894 if (global.truthTable[static_cast<int>(vars.objectType)][output_truth_table_loc] &&
4895 local_sets[p][b].entity_count() > 0) {
4896
4897 T *iv_block_mev = Data(master_values);
4898 size_t entity_count = local_sets[p][b].entity_count();
4899
4900 if (local_mesh[p]
4901 .truthTable[static_cast<int>(vars.objectType)][input_truth_table_loc] > 0) {
4902 int error = ex_get_var(id, time_step + 1, exodus_object_type(vars.objectType),
4903 i + 1, local_sets[p][b].id, entity_count, Data(values));
4904 if (error < 0) {
4905 exodus_error(__LINE__);
4906 }
4907
4908 switch (vars.objectType) {
4909 case Excn::ObjectType::EBLK:
4910 map_element_vars(local_sets[p][b].offset_, global_sets[b].offset_, entity_count,
4911 values, iv_block_mev, proc_loc_elem_to_global);
4912 break;
4913
4914 case Excn::ObjectType::SSET:
4915 map_sideset_vars(local_sets[p][b].offset_, entity_count, values, iv_block_mev);
4916 break;
4917
4918 case Excn::ObjectType::NSET:
4919 map_nodeset_vars(local_sets[p][b], entity_count, global_sets[b].entity_count(),
4920 values, iv_block_mev);
4921 break;
4922
4923 case Excn::ObjectType::EDBLK:
4924 map_edgeblock_vars(local_sets[p][b].offset_, global_sets[b].offset_, entity_count,
4925 values, iv_block_mev, proc_loc_edge_to_global);
4926 break;
4927
4928 case Excn::ObjectType::FABLK:
4929 map_edgeblock_vars(local_sets[p][b].offset_, global_sets[b].offset_, entity_count,
4930 values, iv_block_mev, proc_loc_face_to_global);
4931 break;
4932 default: break;
4933 }
4934 }
4935 }
4936 }
4937 int truth_table_loc = (b * vars.count(InOut::OUT)) + ivar;
4938
4939 if (global.truthTable[static_cast<int>(vars.objectType)][truth_table_loc]) {
4940 int error =
4941 ex_put_var(id_out, time_step_out, exodus_object_type(vars.objectType), ivar + 1,
4942 global_sets[b].id, global_sets[b].entity_count(), Data(master_values));
4943 if (error < 0) {
4944 exodus_error(__LINE__);
4945 }
4946 }
4947 }
4948 }
4949 }
4950 }
4951
4952 template <typename U>
4953 void create_output_truth_table(const Excn::Mesh &global, std::vector<U> &global_sets,
4954 Excn::Variables &vars, std::vector<int> &truth_table)
4955 {
4956 for (size_t b = 0; b < global.count(vars.objectType); b++) {
4957 int bout = global_sets[b].position_;
4958 SMART_ASSERT(bout >= 0);
4959 for (int j = 0; j < vars.count(InOut::OUT); j++) {
4960 int inp_ttable_loc = (b * vars.count(InOut::OUT)) + j;
4961 int out_ttable_loc = (bout * vars.count(InOut::OUT)) + j;
4962 truth_table[out_ttable_loc] =
4963 global.truthTable[static_cast<int>(vars.objectType)][inp_ttable_loc];
4964 }
4965 }
4966 }
4967
4968 template <typename INT>
4969 size_t find_max_entity_count(int part_count, std::vector<Excn::Mesh> &local_mesh,
4970 const Excn::Mesh &global, std::vector<std::vector<Block>> &blocks,
4971 std::vector<std::vector<NodeSet<INT>>> &nodesets,
4972 std::vector<std::vector<SideSet<INT>>> &sidesets,
4973 std::vector<std::vector<EdgeBlock<INT>>> &edgeblocks,
4974 std::vector<std::vector<FaceBlock<INT>>> &faceblocks)
4975 {
4976 size_t max_ent = local_mesh[0].nodeCount;
4977 for (int p = 1; p < part_count; p++) {
4978 if (static_cast<size_t>(local_mesh[p].nodeCount) > max_ent) {
4979 max_ent = local_mesh[p].nodeCount;
4980 }
4981 }
4982
4983 for (int p = 0; p < part_count; p++) {
4984 for (size_t b = 0; b < global.count(Excn::ObjectType::EBLK); b++) {
4985 if (blocks[p][b].entity_count() > max_ent) {
4986 max_ent = blocks[p][b].entity_count();
4987 }
4988 }
4989 }
4990
4991 // Nodesets...
4992 for (int p = 0; p < part_count; p++) {
4993 for (size_t b = 0; b < global.count(Excn::ObjectType::NSET); b++) {
4994 if (nodesets[p][b].entity_count() > max_ent) {
4995 max_ent = nodesets[p][b].entity_count();
4996 }
4997 }
4998 }
4999
5000 // Sidesets...
5001 for (int p = 0; p < part_count; p++) {
5002 for (size_t b = 0; b < global.count(Excn::ObjectType::SSET); b++) {
5003 if (sidesets[p][b].entity_count() > max_ent) {
5004 max_ent = sidesets[p][b].entity_count();
5005 }
5006 }
5007 }
5008
5009 // Edgeblocks...
5010 for (int p = 0; p < part_count; p++) {
5011 for (size_t b = 0; b < global.count(Excn::ObjectType::EDBLK); b++) {
5012 if (edgeblocks[p][b].entity_count() > max_ent) {
5013 max_ent = edgeblocks[p][b].entity_count();
5014 }
5015 }
5016 }
5017
5018 // Faceblocks...
5019 for (int p = 0; p < part_count; p++) {
5020 for (size_t b = 0; b < global.count(Excn::ObjectType::FABLK); b++) {
5021 if (faceblocks[p][b].entity_count() > max_ent) {
5022 max_ent = faceblocks[p][b].entity_count();
5023 }
5024 }
5025 }
5026 return max_ent;
5027 }
5028
5029 template <typename INT>
5030 size_t find_max_global_entity_count(const Excn::Mesh &global, std::vector<Block> &blocks,
5031 std::vector<NodeSet<INT>> &nodesets,
5032 std::vector<SideSet<INT>> &sidesets,
5033 std::vector<EdgeBlock<INT>> &edgeblocks,
5034 std::vector<FaceBlock<INT>> &faceblocks)
5035 {
5036 size_t max_ent = global.nodeCount;
5037
5038 for (size_t b = 0; b < global.count(Excn::ObjectType::EBLK); b++) {
5039 if (blocks[b].entity_count() > max_ent) {
5040 max_ent = blocks[b].entity_count();
5041 }
5042 }
5043
5044 // Nodesets...
5045 for (size_t b = 0; b < global.count(Excn::ObjectType::NSET); b++) {
5046 if (nodesets[b].entity_count() > max_ent) {
5047 max_ent = nodesets[b].entity_count();
5048 }
5049 }
5050
5051 // Sidesets...
5052 for (size_t b = 0; b < global.count(Excn::ObjectType::SSET); b++) {
5053 if (sidesets[b].entity_count() > max_ent) {
5054 max_ent = sidesets[b].entity_count();
5055 }
5056 }
5057
5058 // Edge Blocks...
5059 for (size_t b = 0; b < global.count(Excn::ObjectType::EDBLK); b++) {
5060 if (edgeblocks[b].entity_count() > max_ent) {
5061 max_ent = edgeblocks[b].entity_count();
5062 }
5063 }
5064
5065 // Face Blocks...
5066 for (size_t b = 0; b < global.count(Excn::ObjectType::FABLK); b++) {
5067 if (faceblocks[b].entity_count() > max_ent) {
5068 max_ent = faceblocks[b].entity_count();
5069 }
5070 }
5071 return max_ent;
5072 }
5073 } // namespace