Debian Codegraph @

packages/seacas/applications/epu/EP_ExodusFile.C

raw
1 /*
2 * Copyright(C) 1999-2021, 2023, 2024, 2025 National Technology & Engineering Solutions
3 * of Sandia, LLC (NTESS). Under the terms of Contract DE-NA0003525 with
4 * NTESS, the U.S. Government retains certain rights in this software.
5 *
6 * See packages/seacas/LICENSE for details
7 */
8
9 #include "EP_ExodusFile.h"
10 #include "EP_Internals.h"
11 #include "EP_ParallelDisks.h"
12 #include "EP_SystemInterface.h"
13 #include "open_file_limit.h"
14 #include "smart_assert.h"
15
16 #include <fmt/color.h>
17 #include <fmt/format.h>
18 #include <fmt/ranges.h>
19
20 #include <climits>
21 #include <cstddef>
22 #include <cstdlib>
23 #include <iostream>
24 #include <sstream>
25 #include <stdexcept>
26 #include <string>
27 #include <unistd.h>
28 #include <vector>
29
30 #include <exodusII.h>
31
32 std::vector<int> Excn::ExodusFile::fileids_;
33 std::vector<std::string> Excn::ExodusFile::filenames_;
34 int Excn::ExodusFile::processorCount_ = 0;
35 int Excn::ExodusFile::partCount_ = 0;
36 int Excn::ExodusFile::startPart_ = 0;
37 int Excn::ExodusFile::outputId_ = -1;
38 int Excn::ExodusFile::ioWordSize_ = 0;
39 int Excn::ExodusFile::cpuWordSize_ = 0;
40 int Excn::ExodusFile::mode64bit_ = 0;
41 int Excn::ExodusFile::changeSetCount_ = -1;
42 int Excn::ExodusFile::activeChangeSet_ = 0;
43 std::string Excn::ExodusFile::outputFilename_;
44 bool Excn::ExodusFile::keepOpen_ = false;
45 bool Excn::ExodusFile::verifyValidFile_ = false;
46 bool Excn::ExodusFile::onlySelectedChangeSet_ = false;
47 int Excn::ExodusFile::maximumNameLength_ = 32;
48
49 namespace {
50 void create_output_change_sets(const Excn::SystemInterface &si)
51 {
52 int save = Excn::ExodusFile::set_active_change_set(0);
53 int output_id = Excn::ExodusFile::output();
54
55 Excn::ExodusFile::set_active_change_set(1);
56 int exoid = Excn::ExodusFile(0);
57 int group_name_length = ex_inquire_int(exoid, EX_INQ_GROUP_NAME_LEN);
58 std::vector<char> group_name(group_name_length + 1, '\0');
59
60 fmt::print("Output file id = {}\n", output_id);
61 for (int i = 0; i < Excn::ExodusFile::get_change_set_count(); i++) {
62 int idum = 0;
63 float rdum = 0.0;
64 // Get name of this group...
65 int ierr = ex_inquire(exoid + i, EX_INQ_GROUP_NAME, &idum, &rdum, group_name.data());
66 if (ierr != EX_NOERR) {
67 throw std::runtime_error(fmt::format(
68 "ERROR: (EPU) Could not get name for group {} in input file - exiting\n", i + 1));
69 }
70 int cs_id = ex_create_group(output_id, group_name.data());
71 if (cs_id <= 0) {
72 throw std::runtime_error(fmt::format(
73 "ERROR: (EPU) Could not create group {} in output file.\n", fmt::join(group_name, "")));
74 }
75 if ((si.debug() & 1024) != 0) {
76 fmt::print("Change set {} is {} with exoid {}\n", i + 1, group_name.data(), cs_id);
77 }
78 }
79 Excn::ExodusFile::set_active_change_set(save);
80 }
81 } // namespace
82
83 Excn::ExodusFile::ExodusFile(int processor) : myProcessor_(processor)
84 {
85 SMART_ASSERT(processor < processorCount_)(processor)(processorCount_);
86 SMART_ASSERT(fileids_.size() == (size_t)processorCount_);
87 if (!keepOpen_ && processor != 0) {
88 float version = 0.0;
89 int cpu_word_size = cpuWordSize_;
90 int io_word_size_var = ioWordSize_;
91 int mode = EX_READ;
92 mode |= mode64bit_;
93
94 fileids_[processor] =
95 ex_open(filenames_[processor].c_str(), mode, &cpu_word_size, &io_word_size_var, &version);
96 if (fileids_[processor] < 0) {
97 throw std::runtime_error(
98 fmt::format("ERROR: (EPU) Cannot open file '{}' - exiting\n", filenames_[processor]));
99 }
100 ex_set_max_name_length(fileids_[processor], maximumNameLength_);
101
102 // Check for change_sets...
103 set_change_set_count(fileids_[processor]);
104
105 SMART_ASSERT(io_word_size_var == ioWordSize_);
106 SMART_ASSERT(cpu_word_size == cpuWordSize_);
107 }
108 }
109
110 void Excn::ExodusFile::set_change_set_count(int exoid)
111 {
112 int num_change_sets = ex_inquire_int(exoid, EX_INQ_NUM_CHILD_GROUPS);
113 if (changeSetCount_ < 0) {
114 changeSetCount_ = num_change_sets;
115 if (activeChangeSet_ >= changeSetCount_) {
116 auto error =
117 fmt::format("ERROR: (EPU) Selected Change set {} exceeds change set count {} - exiting\n",
118 activeChangeSet_ + 1, changeSetCount_);
119 throw std::runtime_error(error);
120 }
121 }
122 else {
123 if (changeSetCount_ != num_change_sets) {
124 throw std::runtime_error(
125 "ERROR: (EPU) Inconsistent change set count in one or more files - exiting\n");
126 }
127 }
128 }
129
130 int Excn::ExodusFile::output()
131 {
132 SMART_ASSERT(outputId_ >= 0);
133 return outputId_ + (onlySelectedChangeSet_ ? 0 : activeChangeSet_);
134 }
135
136 Excn::ExodusFile::operator int() const
137 {
138 SMART_ASSERT(fileids_[myProcessor_] >= 0);
139 return fileids_[myProcessor_] + activeChangeSet_;
140 }
141
142 Excn::ExodusFile::~ExodusFile()
143 {
144 try {
145 if (!keepOpen_ && myProcessor_ != 0) {
146 ex_close(fileids_[myProcessor_]);
147 fileids_[myProcessor_] = -1;
148 }
149 }
150 catch (...) {
151 }
152 }
153
154 void Excn::ExodusFile::close_all()
155 {
156 for (int p = 0; p < partCount_; p++) {
157 if (fileids_[p] >= 0) {
158 ex_close(fileids_[p]);
159 fileids_[p] = -1;
160 }
161 }
162 if (outputId_ >= 0) {
163 ex_close(outputId_);
164 outputId_ = -1;
165 }
166
167 // We have had some issues with file corruption. Everything seems to be
168 // working fine, but when the file is opened later after EPU has completed,
169 // one or more of the output files (in subcyle mode) report being invalid.
170 // Here, if the user requests, we try to reopen the file and report an
171 // error if there is a problem. This might still not catch all bad files,
172 // but will hopefully catch some at the point that it happens...
173 if (verifyValidFile_) {
174 float version = 0.0;
175 int cpu_word_size = cpuWordSize_;
176 int io_word_size_var = ioWordSize_;
177 int mode = EX_READ;
178 mode |= mode64bit_;
179 int exoid = ex_open(outputFilename_.c_str(), mode, &cpu_word_size, &io_word_size_var, &version);
180 if (exoid < 0) {
181 ex_get_err(nullptr, nullptr, &exoid);
182 fmt::print(stderr, fmt::fg(fmt::color::red),
183 "EPU: Exodus error ({}) {}.\n"
184 "Output File verification failed for '{}'. Could not reopen output file after "
185 "closing it\n",
186 exoid, ex_strerror(exoid), outputFilename_);
187 }
188 else {
189 ex_close(exoid);
190 }
191 }
192 }
193
194 void Excn::ExodusFile::unlink_input_files()
195 {
196 fmt::print("\n\tUnlinking {}\n\t ...", filenames_[0]);
197 for (int p = 0; p < partCount_; p++) {
198 unlink(filenames_[p].c_str());
199 }
200 fmt::print("\n\tUnlinking {}\n\n", filenames_[partCount_ - 1]);
201 }
202
203 void Excn::ExodusFile::handle_temporary_files(bool delete_them)
204 {
205 for (int p = 0; p < partCount_; p++) {
206 if (fileids_[p] >= 0) {
207 ex_close(fileids_[p]);
208 if (delete_them) {
209 unlink(filenames_[p].c_str());
210 }
211 fileids_[p] = -1;
212 }
213 }
214 if (outputId_ >= 0) {
215 ex_close(outputId_);
216 outputId_ = -1;
217 }
218 }
219
220 void Excn::ExodusFile::initialize(const SystemInterface &si, int start_part, int part_count,
221 int cycle, bool joining_subcycle)
222 {
223 processorCount_ = si.processor_count(); // Total number processors
224 partCount_ = part_count; // Total parts we are processing
225 startPart_ = start_part; // Which one to start with
226 SMART_ASSERT(partCount_ + startPart_ <= processorCount_)(partCount_)(startPart_)(processorCount_);
227
228 activeChangeSet_ = si.selected_change_set();
229 if (si.selected_change_set() > 0) {
230 onlySelectedChangeSet_ = true;
231 ;
232 }
233
234 // EPU always wants entity (block, set, map) ids as 64-bit quantities...
235 mode64bit_ = EX_IDS_INT64_API;
236 if (si.int64()) {
237 mode64bit_ |= EX_ALL_INT64_API;
238
239 // For output...
240 mode64bit_ |= EX_ALL_INT64_DB;
241 }
242
243 verifyValidFile_ = si.verify_valid_file();
244
245 // See if we can keep files open
246 int max_files = open_file_limit() - 1; // We also have an output exodus file.
247 if (partCount_ <= max_files) {
248 keepOpen_ = true;
249 if (cycle == 0) {
250 if ((si.debug() & 1) != 0) {
251 fmt::print("Files kept open... (Max open = {})\n\n", max_files);
252 }
253 }
254 }
255 else {
256 keepOpen_ = false;
257 if (cycle == 0) {
258 fmt::print("Single file mode... (Max open = {})\n"
259 "Consider using the -subcycle option for faster execution...\n\n",
260 max_files);
261 }
262 }
263
264 fileids_.resize(processorCount_);
265 filenames_.resize(processorCount_);
266
267 std::string file_prefix = si.basename();
268 std::string exodus_suffix = si.exodus_suffix();
269
270 std::string root_dir = si.root_dir();
271 std::string sub_dir = si.sub_dir();
272
273 ParallelDisks disks;
274
275 float version = 0.0;
276
277 // create exo names
278 for (int p = 0; p < partCount_; p++) {
279 std::string name = file_prefix;
280 if (!exodus_suffix.empty()) {
281 name += "." + exodus_suffix;
282 }
283
284 int proc = p + startPart_;
285 if (joining_subcycle) {
286 name = si.output_filename();
287 disks.rename_file_for_mp("", "", name, proc, processorCount_);
288 }
289 else {
290 disks.rename_file_for_mp(root_dir, sub_dir, name, proc, processorCount_);
291 }
292 filenames_[p] = name;
293
294 if (p == 0) {
295 int cpu_word_size = sizeof(float);
296 int io_word_size_var = 0;
297 int mode = EX_READ;
298 mode |= mode64bit_;
299 int exoid = ex_open(filenames_[p].c_str(), mode, &cpu_word_size, &io_word_size_var, &version);
300 if (exoid < 0) {
301 fmt::print(stderr, fmt::fg(fmt::color::red), "Cannot open file '{}'\n", filenames_[p]);
302 throw std::runtime_error("ERROR: (EPU) Problem initializing input and/or output files.\n");
303 }
304
305 // Check for change_sets...
306 set_change_set_count(exoid);
307
308 int int64db = ex_int64_status(exoid) & EX_ALL_INT64_DB;
309 if (int64db != 0) {
310 // If anything stored on input db as 64-bit int, then output db will have
311 // everything stored as 64-bit ints and all API functions will use 64-bit
312 mode64bit_ |= EX_ALL_INT64_API;
313 mode64bit_ |= EX_ALL_INT64_DB;
314 }
315
316 int max_name = static_cast<int>(ex_inquire_int(exoid, EX_INQ_DB_MAX_USED_NAME_LENGTH));
317 if (max_name > maximumNameLength_) {
318 maximumNameLength_ = max_name;
319 }
320
321 ex_close(exoid);
322
323 if (io_word_size_var < static_cast<int>(sizeof(float))) {
324 io_word_size_var = sizeof(float);
325 }
326
327 ioWordSize_ = io_word_size_var;
328 cpuWordSize_ = io_word_size_var;
329 }
330
331 if (keepOpen_ || p == 0) {
332 int io_word_size_var = 0;
333 int mode = EX_READ;
334 // All entity ids (block, sets) are read/written as 64-bit...
335 mode |= mode64bit_;
336 fileids_[p] =
337 ex_open(filenames_[p].c_str(), mode, &cpuWordSize_, &io_word_size_var, &version);
338 if (fileids_[p] < 0) {
339 fmt::print(stderr, fmt::fg(fmt::color::red), "Cannot open file '{}'\n", filenames_[p]);
340 throw std::runtime_error("ERROR: (EPU) Problem initializing input and/or output files.\n");
341 }
342 ex_set_max_name_length(fileids_[p], maximumNameLength_);
343 SMART_ASSERT(ioWordSize_ == io_word_size_var)(ioWordSize_)(io_word_size_var);
344 }
345
346 if (((si.debug() & 64) != 0) || p == 0 || p == partCount_ - 1) {
347 fmt::print("[{}] Input({}): '{}'\n", cycle, p, name);
348 if (((si.debug() & 64) == 0) && p == 0) {
349 fmt::print("...\n");
350 }
351 }
352 }
353
354 if ((mode64bit_ & EX_ALL_INT64_DB) != 0) {
355 if (cycle == 0) {
356 fmt::print("Input files contain 8-byte integers.\n");
357 }
358 si.set_int64();
359 }
360 }
361
362 // `cs` is the 0-based change-set to select
363 int Excn::ExodusFile::set_active_change_set(int cs)
364 {
365 int save = activeChangeSet_;
366
367 if (changeSetCount_ < 0) {
368 throw std::runtime_error(
369 "ERROR: (EPU) Cannot set active change set before changes sets have been queried.\n");
370 }
371 if (cs <= changeSetCount_) {
372 activeChangeSet_ = cs;
373 }
374 else {
375 throw std::runtime_error(
376 fmt::format("ERROR: (EPU) Active change set index {} exceeds change set count {}.\n", cs,
377 changeSetCount_));
378 }
379 return save;
380 }
381
382 void Excn::ExodusFile::create_output(const SystemInterface &si, int cycle)
383 // Create output file...
384 {
385 std::string curdir = si.cwd();
386 std::string file_prefix = si.basename();
387 std::string output_suffix = si.output_suffix();
388
389 outputFilename_ = std::move(file_prefix);
390 if (!output_suffix.empty()) {
391 outputFilename_ += "." + output_suffix;
392 }
393
394 if (!curdir.empty() && !Excn::is_path_absolute(outputFilename_)) {
395 outputFilename_ = curdir + "/" + outputFilename_;
396 }
397
398 si.set_output_filename(outputFilename_);
399
400 if (si.subcycle() > 1) {
401 Excn::ParallelDisks::Create_IO_Filename(outputFilename_, cycle, si.subcycle());
402 }
403
404 // See if output file should be opened in netcdf4 format...
405 // Did user specify it via -netcdf4 or -large_model argument...
406 int mode = 0;
407
408 if (changeSetCount_ > 1 && !onlySelectedChangeSet_) {
409 mode |= EX_NETCDF4;
410 mode |= EX_NOCLASSIC;
411 }
412 else if (si.compress_data() > 0 || si.szip()) {
413 // Force netcdf-4 if compression is specified...
414 mode |= EX_NETCDF4;
415 }
416 else if (si.use_netcdf4()) {
417 mode |= EX_NETCDF4;
418 }
419 else if (si.use_netcdf5()) {
420 mode |= EX_64BIT_DATA;
421 }
422 else if (ex_large_model(fileids_[0]) == 1) {
423 mode |= EX_LARGE_MODEL;
424 }
425
426 mode |= mode64bit_;
427
428 if (si.int64()) {
429 mode |= EX_ALL_INT64_DB;
430 mode |= EX_ALL_INT64_API;
431 }
432
433 if (si.append()) {
434 fmt::print("[{}] Output: '{}' (appending)\n", cycle, outputFilename_);
435 float version = 0.0;
436 mode |= EX_WRITE;
437 outputId_ = ex_open(outputFilename_.c_str(), mode, &cpuWordSize_, &ioWordSize_, &version);
438 }
439 else {
440 mode |= EX_CLOBBER;
441 fmt::print("[{}] Output: '{}'\n", cycle, outputFilename_);
442 outputId_ = ex_create(outputFilename_.c_str(), mode, &cpuWordSize_, &ioWordSize_);
443 }
444 if (outputId_ < 0) {
445 fmt::print(stderr, fmt::fg(fmt::color::red), "Cannot open file '{}'\n", outputFilename_);
446 throw std::runtime_error("ERROR: (EPU) Problem creating output file.\n");
447 }
448
449 if (si.compress_data() > 0 || si.szip()) {
450 if (si.szip()) {
451 ex_set_option(outputId_, EX_OPT_COMPRESSION_TYPE, EX_COMPRESS_SZIP);
452 }
453 else if (si.zlib()) {
454 ex_set_option(outputId_, EX_OPT_COMPRESSION_TYPE, EX_COMPRESS_ZLIB);
455 }
456 else if (si.zstd()) {
457 ex_set_option(outputId_, EX_OPT_COMPRESSION_TYPE, EX_COMPRESS_ZSTD);
458 }
459 else if (si.bz2()) {
460 ex_set_option(outputId_, EX_OPT_COMPRESSION_TYPE, EX_COMPRESS_BZ2);
461 }
462 ex_set_option(outputId_, EX_OPT_COMPRESSION_LEVEL, si.compress_data());
463 ex_set_option(outputId_, EX_OPT_COMPRESSION_SHUFFLE, 1);
464
465 if (si.quantize()) {
466 ex_set_option(outputId_, EX_OPT_QUANTIZE_NSD, si.quantize_nsd());
467 }
468 }
469
470 // Check whether input file(s) have changesets. If they do and the `selected_change_set` is
471 // not `0`, then we are combining all change sets on the input to the output. Create them
472 // now...
473 if (changeSetCount_ > 1 && !onlySelectedChangeSet_) {
474 // Get the names of the change sets on the input file and create them on output file...
475 create_output_change_sets(si);
476 }
477
478 // EPU Can add a name of "processor_id_epu" which is 16 characters long.
479 // Make sure maximumNameLength_ is at least that long...
480 if (maximumNameLength_ < 16) {
481 maximumNameLength_ = 16;
482 }
483 ex_set_option(outputId_, EX_OPT_MAX_NAME_LENGTH, maximumNameLength_);
484
485 int int_size = si.int64() ? 8 : 4;
486 if (cycle == 0) {
487 fmt::print("IO Word sizes: {} bytes floating point and {} bytes integer.\n", ioWordSize_,
488 int_size);
489 }
490 }