Debian Codegraph @

packages/seacas/applications/cpup/CP_SystemInterface.C

raw
1 /*
2 * Copyright(C) 1999-2024 National Technology & Engineering Solutions
3 * of Sandia, LLC (NTESS). Under the terms of Contract DE-NA0003525 with
4 * NTESS, the U.S. Government retains certain rights in this software.
5 *
6 * See packages/seacas/LICENSE for details
7 */
8 #include "CP_SystemInterface.h"
9 #include "CP_Version.h" // for qainfo
10 #include "FileInfo.h"
11 #include "GetLongOpt.h" // for GetLongOption, etc
12 #include "Ioss_CodeTypes.h"
13 #include "Ioss_Utils.h"
14 #include "SL_tokenize.h" // for tokenize
15 #include <algorithm> // for sort, transform
16 #include <cctype> // for tolower
17 #include <copyright.h>
18 #include <cstdlib> // for strtol, abs, exit, strtoul, etc
19 #include <cstring> // for strchr, strlen
20 #include <fmt/ostream.h>
21 #include <glob.h>
22 #include <sstream>
23 #include <stdexcept>
24 #include <string> // for string, char_traits, etc
25 #include <term_width.h>
26 #include <unistd.h>
27 #include <utility> // for pair, make_pair
28 #include <vector> // for vector
29
30 #if defined(WIN32) || defined(__WIN32__) || defined(_WIN32) || defined(_MSC_VER) || \
31 defined(__MINGW32__) || defined(_WIN64) || defined(__MINGW64__)
32 #define __SUP_WINDOWS__ 1
33 #include <direct.h>
34 #endif
35
36 #if !defined(__SUP_WINDOWS__)
37 #include <dirent.h>
38 #endif
39
40 namespace {
41 bool str_equal(const std::string &s1, const std::string &s2)
42 {
43 return (s1.size() == s2.size()) &&
44 std::equal(s1.begin(), s1.end(), s2.begin(),
45 [](char a, char b) { return std::tolower(a) == std::tolower(b); });
46 }
47
48 void parse_variable_names(const char *tokens, Cpup::StringVector *variable_list);
49 std::string find_matching_file(const std::string &path, const std::string &basename);
50 } // namespace
51
52 Cpup::SystemInterface::SystemInterface(int rank) : myRank_(rank) { enroll_options(); }
53
54 void Cpup::SystemInterface::enroll_options()
55 {
56 options_.usage("[options] basename");
57
58 options_.enroll("auto", GetLongOption::NoValue,
59 "Automatically set Root, Proc, Ext from filename 'Root/basename.ext.#p.00'.",
60 nullptr, nullptr, true);
61 options_.enroll("variables", GetLongOption::MandatoryValue,
62 "Comma-separated list of variables (fields) to be joined or ALL or NONE.",
63 nullptr);
64 options_.enroll("steps", GetLongOption::MandatoryValue,
65 "Specify subset of timesteps to transfer to output file.\n"
66 "\t\tFormat is beg:end:step. 1:10:2 --> 1,3,5,7,9\n"
67 "\t\tIf the 'beg' or 'end' is < 0, then it is the \"-Nth\" step...\n"
68 "\t\t-1 is \"first last\" or last, -3 is \"third last\"\n"
69 "\t\tTo copy just the last 3 steps, do: `-steps -3:-1`\n"
70 "\t\tEnter LAST for last step",
71 "1:", nullptr, true);
72
73 options_.enroll("extension", GetLongOption::MandatoryValue,
74 "CGNS database extension for the input files", "cgns");
75
76 options_.enroll("output", GetLongOption::MandatoryValue, "filename for output CGNS database",
77 nullptr);
78
79 options_.enroll("output_extension", GetLongOption::MandatoryValue,
80 "CGNS database extension for the output file", nullptr);
81
82 options_.enroll("processor_count", GetLongOption::MandatoryValue, "Number of processors", "1");
83
84 options_.enroll("current_directory", GetLongOption::MandatoryValue, "Current Directory", ".");
85
86 options_.enroll("Root_directory", GetLongOption::MandatoryValue, "Root directory", nullptr);
87
88 options_.enroll("Subdirectory", GetLongOption::MandatoryValue,
89 "subdirectory containing input cgns files", nullptr, nullptr, true);
90
91 #if 0
92 options_.enroll("subcycle", GetLongOption::OptionalValue,
93 "Subcycle. Create $val subparts if $val is specified.\n"
94 "\t\tOtherwise, create multiple parts each of size 'Part_count'.\n"
95 "\t\tThe subparts can then be joined by a subsequent run of cpup.\n"
96 "\t\tUseful if the maximum number of open files is less\n"
97 "\t\tthan the processor count.",
98 nullptr, "0");
99
100 options_.enroll("join_subcycles", GetLongOption::NoValue,
101 "If -subcycle is specified, then after the subcycle files are processed,\n"
102 "\t\trun cpup one more time and join the subcycle files into a single file.",
103 nullptr);
104
105 options_.enroll("Part_count", GetLongOption::MandatoryValue,
106 "How many pieces (files) of the model should be joined.", "0");
107
108 options_.enroll("start_part", GetLongOption::MandatoryValue, "Start with piece {L} (file)", "0");
109
110 options_.enroll("cycle", GetLongOption::MandatoryValue,
111 "Cycle number. If subcycle # is specified, then only execute\n"
112 "\t\tcycle $val ($val < #). The cycle number is 0-based.",
113 "-1");
114
115 options_.enroll("keep_temporary", GetLongOption::NoValue,
116 "If -join_subcycles is specified, then after joining the subcycle files,\n"
117 "\t\tthey are automatically deleted unless -keep_temporary is specified.",
118 nullptr);
119
120 options_.enroll(
121 "verify_valid_file", GetLongOption::NoValue,
122 "Reopen the output file right after closing it to verify that the file is valid.\n"
123 "\t\tThis tries to detect file corruption immediately instead of later. Mainly useful in "
124 "large subcycle runs.",
125 nullptr);
126
127 options_.enroll("append", GetLongOption::NoValue,
128 "Append to database instead of opening a new database.\n"
129 "\t\tTimestep transfer will start after last timestep on database",
130 nullptr);
131
132 options_.enroll("add_processor_id", GetLongOption::NoValue,
133 "Add 'processor_id' element variable to the output file which shows the\n"
134 "\t\tprocessor that an element was on in the decomposed mesh.\n"
135 "\t\tCan be used by SLICE or auto-decomp to reproduce decomposition.",
136 nullptr);
137
138 options_.enroll("add_map_processor_id", GetLongOption::NoValue,
139 "Add 'processor_id' element map to the output file which shows the\n"
140 "\t\tprocessor that an element was on in the decomposed mesh.\n"
141 "\t\tCan be used by SLICE or auto-decomp to reproduce decomposition.",
142 nullptr, nullptr, true);
143
144 options_.enroll("gvar", GetLongOption::MandatoryValue,
145 "Comma-separated list of global variables to be joined or ALL or NONE.", nullptr);
146
147 options_.enroll("evar", GetLongOption::MandatoryValue,
148 "Comma-separated list of element variables to be joined or ALL or NONE.\n"
149 "\t\tVariables can be limited to certain blocks by appending a\n"
150 "\t\tcolon followed by the block id. E.g. -evar sigxx:10:20",
151 nullptr);
152
153 options_.enroll("nvar", GetLongOption::MandatoryValue,
154 "Comma-separated list of nodal variables to be joined or ALL or NONE.", nullptr);
155
156 options_.enroll("nsetvar", GetLongOption::MandatoryValue,
157 "Comma-separated list of nodeset variables to be joined or ALL or NONE.",
158 nullptr);
159
160 options_.enroll("ssetvar", GetLongOption::MandatoryValue,
161 "Comma-separated list of sideset variables to be joined or ALL or NONE.",
162 nullptr);
163
164 options_.enroll("omit_nodesets", GetLongOption::NoValue,
165 "Don't transfer nodesets to output file.", nullptr);
166
167 options_.enroll("omit_sidesets", GetLongOption::NoValue,
168 "Don't transfer sidesets to output file.", nullptr, nullptr, true);
169
170 options_.enroll("sum_shared_nodes", GetLongOption::NoValue,
171 "The nodal results data on all shared nodes (nodes on processor boundaries)\n"
172 "\t\twill be the sum of the individual nodal results data on each shared node.\n"
173 "\t\tThe default behavior assumes that the values are equal.",
174 nullptr);
175
176 options_.enroll("max_open_files", GetLongOption::MandatoryValue,
177 "For testing auto subcycle only. Sets file limit that triggers auto subcycling.",
178 "0");
179
180 #endif
181 options_.enroll(
182 "minimize_open_files", GetLongOption::NoValue,
183 "Keep only one input file open at a time.\n"
184 "\t\tUsed when number of files exceeds the maximum number of open files on a system.",
185 nullptr);
186
187 options_.enroll("width", GetLongOption::MandatoryValue, "Width of output screen, default = 80",
188 nullptr);
189
190 options_.enroll("debug", GetLongOption::MandatoryValue,
191 "debug level (values are or'd)\n"
192 "\t\t 1 = Timestamp output.\n"
193 "\t\t 2 = Output region summary for each input part mesh.\n"
194 "\t\t 4 = Verbose Structured block information.",
195 "0");
196 options_.enroll("copyright", GetLongOption::NoValue, "Show copyright and license data.", nullptr);
197 options_.enroll("help", GetLongOption::NoValue, "Print this summary and exit", nullptr);
198
199 options_.enroll("version", GetLongOption::NoValue, "Print version and exit", nullptr, nullptr);
200 }
201
202 bool Cpup::SystemInterface::parse_options(int argc, char **argv)
203 {
204 // Get options from environment variable also...
205 char *options = getenv("CPUP_OPTIONS");
206 if (options != nullptr) {
207 if (myRank_ == 0) {
208 fmt::print(
209 "\nThe following options were specified via the CPUP_OPTIONS environment variable:\n"
210 "\t{}\n\n",
211 options);
212 }
213 options_.parse(options, GetLongOption::basename(*argv));
214 }
215
216 int option_index = options_.parse(argc, argv);
217 if (option_index < 1) {
218 throw std::runtime_error("ERROR: (CPUP) No arguments specified.");
219 }
220
221 if (options_.retrieve("help") != nullptr) {
222 if (myRank_ == 0) {
223 options_.usage();
224 fmt::print("\n\tCan also set options via CPUP_OPTIONS environment variable.\n\n"
225 "\n\tDocumentation: "
226 "https://sandialabs.github.io/seacas-docs/sphinx/html/index.html#cpup\n"
227 "\tWrites: current_directory/basename.output_extension\n"
228 "\tReads: root/sub/basename.extension.#p.0 to\n"
229 "\t\troot/sub/basename.extension.#p.#p-1\n"
230 "\n\t->->-> Send email to [email protected] for cpup support.<-<-<-\n");
231 }
232 return false;
233 }
234
235 if (options_.retrieve("version") != nullptr) {
236 // Version is printed up front, just exit...
237 return false;
238 }
239
240 inExtension_ = options_.get_option_value("extension", inExtension_);
241 outExtension_ = options_.get_option_value("output_extension", outExtension_);
242 outputFilename_ = options_.get_option_value("output", outputFilename_);
243 // partCount_ = options_.get_option_value("Part_count", partCount_);
244 // startPart_ = options_.get_option_value("start_part", startPart_);
245 // maxOpenFiles_ = options_.get_option_value("max_open_files", maxOpenFiles_);
246 cwd_ = options_.get_option_value("current_directory", cwd_);
247 rootDirectory_ = options_.get_option_value("Root_directory", rootDirectory_);
248 subDirectory_ = options_.get_option_value("Subdirectory", subDirectory_);
249 debugLevel_ = options_.get_option_value("debug", debugLevel_);
250
251 screenWidth_ = options_.get_option_value("width", term_width());
252
253 {
254 const char *temp = options_.retrieve("steps");
255 if (temp != nullptr) {
256 parse_step_option(temp);
257 }
258 }
259
260 {
261 const char *temp = options_.retrieve("variables");
262 parse_variable_names(temp, &varNames_);
263 }
264
265 #if 0
266 {
267 const char *temp = options_.retrieve("nvar");
268 parse_variable_names(temp, &nodeVarNames_);
269 }
270
271 {
272 const char *temp = options_.retrieve("evar");
273 parse_variable_names(temp, &elemVarNames_);
274 }
275
276 {
277 const char *temp = options_.retrieve("nsetvar");
278 parse_variable_names(temp, &nsetVarNames_);
279 }
280
281 {
282 const char *temp = options_.retrieve("ssetvar");
283 parse_variable_names(temp, &ssetVarNames_);
284 }
285
286 addProcessorIdField_ = options_.retrieve("add_processor_id") != nullptr;
287 addProcessorIdMap_ = options_.retrieve("add_map_processor_id") != nullptr;
288
289 append_ = options_.retrieve("append") != nullptr;
290
291 append_ = options_.retrieve("append") != nullptr;
292 //subcycle_ = options_.get_option_value("subcycle", subcycle_);
293 cycle_ = options_.get_option_value("cycle", cycle_);
294 subcycleJoin_ = options_.retrieve("join_subcycles") != nullptr;
295 keepTemporary_ = options_.retrieve("keep_temporary") != nullptr;
296 verifyValidFile_ = options_.retrieve("verify_valid_file") != nullptr;
297
298 omitNodesets_ = options_.retrieve("omit_nodesets") != nullptr;
299 omitSidesets_ = options_.retrieve("omit_sidesets") != nullptr;
300 #endif
301
302 minimizeOpenFiles_ = options_.retrieve("minimize_open_files") != nullptr;
303
304 if (options_.retrieve("copyright") != nullptr) {
305 if (myRank_ == 0) {
306 fmt::print("{}", copyright("2010-2022"));
307 }
308 return false;
309 }
310
311 // Parse remaining options as directory paths.
312 if (option_index < argc) {
313 basename_ = argv[option_index];
314
315 if (options_.retrieve("auto") != nullptr) {
316 // Determine Root, Proc, Extension, and Basename automatically
317 // by parsing the basename_ entered by the user. Assumed to be
318 // in the form: "/directory/sub/basename.ext.#proc.34"
319 FileInfo file(basename_);
320 auto path = file.pathname();
321 if (path.empty()) {
322 path = ".";
323 }
324 #if defined(__SUP_WINDOWS__)
325 rootDirectory_ = _fullpath(nullptr, path.c_str(), _MAX_PATH);
326 #else
327 char *tmp = ::realpath(path.c_str(), nullptr);
328 if (tmp != nullptr) {
329 rootDirectory_ = std::string(tmp);
330 free(tmp);
331 }
332 #endif
333
334 basename_ = file.tailname();
335 if (basename_.empty()) {
336 std::ostringstream errmsg;
337 fmt::print(
338 errmsg,
339 "\nERROR: (CPUP) If the '-auto' option is specified, the basename must specify an "
340 "existing filename or portion of a base of a filename (no rank/proc count).\n"
341 " The entered basename ('{}') does not contain a filename.\n",
342 basename_);
343 throw std::runtime_error(errmsg.str());
344 }
345 bool success = decompose_filename(basename_);
346 if (!success) {
347 // See if we can find files that match the basename and take the first match as the "new"
348 // basename...
349 std::string candidate = find_matching_file(rootDirectory_, basename_);
350 if (!candidate.empty()) {
351 basename_ = candidate;
352 success = decompose_filename(basename_);
353 if (!success) {
354 std::ostringstream errmsg;
355 fmt::print(
356 errmsg,
357 "\nERROR: (CPUP) If the '-auto' option is specified, the basename must specify an "
358 "existing filename or a basename (no rank/proc count).\n"
359 " The entered basename ('{}') does not contain an extension or processor "
360 "count.\n",
361 basename_);
362 throw std::runtime_error(errmsg.str());
363 }
364 }
365 }
366 auto_ = true;
367 if (myRank_ == 0) {
368 fmt::print("\nThe following options were determined automatically:\n"
369 "\t basename = '{}'\n"
370 "\t-processor_count {}\n"
371 "\t-extension {}\n"
372 "\t-Root_directory {}\n\n",
373 basename_, processorCount_, inExtension_, rootDirectory_);
374 }
375 }
376 }
377 else {
378 throw std::runtime_error("\nERROR: (CPUP) basename not specified\n");
379 }
380
381 #if 0
382 // Check that subcycle count does not match processor count --
383 // in that case the existing files will be overwritten.
384 if (processorCount_ <= subcycle_) {
385 if (myRank_ == 0) {
386 fmt::print(stderr,
387 "\nERROR: (CPUP) Invalid subcycle count specified: '{}'."
388 "\n Must be less than processor count '{}'.\n\n",
389 subcycle_, processorCount_);
390 }
391 return false;
392 }
393
394 // If subcycle is specified, but not part_count, then calculate partCount_
395 if (partCount_ <= 0 && subcycle_ > 0) {
396 partCount_ = processorCount_ / subcycle_;
397 }
398 #endif
399 return true;
400 }
401
402 std::string Cpup::SystemInterface::output_filename() const
403 {
404 if (outputFilename_.empty()) {
405 outputFilename_ = basename();
406 if (!output_suffix().empty()) {
407 outputFilename_ += "." + output_suffix();
408 }
409
410 if (!cwd().empty() && !Ioss::Utils::is_path_absolute(outputFilename_)) {
411 outputFilename_ = cwd() + "/" + outputFilename_;
412 }
413 set_output_filename(outputFilename_);
414 }
415 return outputFilename_;
416 }
417
418 void Cpup::SystemInterface::dump(std::ostream & /*unused*/) const {}
419
420 std::string Cpup::SystemInterface::output_suffix() const
421 {
422 if (outExtension_.empty()) {
423 return inExtension_;
424 }
425 return outExtension_;
426 }
427
428 void Cpup::SystemInterface::show_version(int rank)
429 {
430 if (rank == 0) {
431 fmt::print("{}\n"
432 "\tCGNS Parallel Unification Program\n"
433 "\t(Version: {}) Modified: {}\n",
434 qainfo[0], qainfo[1], qainfo[2]);
435 }
436 }
437
438 void Cpup::SystemInterface::parse_step_option(const char *tokens)
439 {
440 //: The defined formats for the count attribute are:<br>
441 //: <ul>
442 //: <li><missing> -- default -- 1 <= count <= oo (all steps)</li>
443 //: <li>"X" -- X <= count <= X (just step X) LAST for last step.</li>
444 //: <li>"X:Y" -- X to Y by 1</li>
445 //: <li>"X:" -- X to oo by 1</li>
446 //: <li>":Y" -- 1 to Y by 1</li>
447 //: <li>"::Z" -- 1 to oo by Z</li>
448 //: </ul>
449 //: The step must always be > 0
450 //: If the 'from' or 'to' is < 0, then it is the "-Nth" step...
451 //: -1 is "first last" or last
452 //: -4 is "fourth last step"
453 //: To copy just the last 3 steps, do: `-steps -3:-1`
454
455 // Break into tokens separated by ":"
456
457 // Default is given in constructor above...
458
459 if (tokens != nullptr) {
460 if (strchr(tokens, ':') != nullptr) {
461 // The string contains a separator
462
463 std::array<int, 3> vals{stepMin_, stepMax_, stepInterval_};
464
465 int j = 0;
466 for (auto &val : vals) {
467 // Parse 'i'th field
468 char tmp_str[128];
469
470 int k = 0;
471 while (tokens[j] != '\0' && tokens[j] != ':') {
472 tmp_str[k++] = tokens[j++];
473 }
474
475 tmp_str[k] = '\0';
476 if (strlen(tmp_str) > 0) {
477 val = strtol(tmp_str, nullptr, 0);
478 }
479
480 if (tokens[j++] == '\0') {
481 break; // Reached end of string
482 }
483 }
484 stepMin_ = vals[0];
485 stepMax_ = vals[1];
486 stepInterval_ = abs(vals[2]); // step is always positive...
487 }
488 else if (str_equal("LAST", tokens)) {
489 stepMin_ = stepMax_ = -1;
490 }
491 else {
492 // Does not contain a separator, min == max
493 stepMin_ = stepMax_ = strtol(tokens, nullptr, 0);
494 }
495 }
496 }
497
498 bool Cpup::SystemInterface::decompose_filename(const std::string &cs)
499 {
500 std::string s(cs);
501 // Decompose the input string 's' into processor count, extension, basename, and root.
502 // Input string should be of the form:
503 // "root/basename.ext.proc.nn"
504 // 'root/' is optional and is all characters preceding the last (if any) '/';
505 // 'basename' can contain multiple '.'
506
507 // NOTE: This used to use the tokenize function, but that didn't work since we need
508 // to handle leading and embedded '..' which tokenize threw away...
509
510 // Get rid of the 'nn' which is not used at this time...
511 size_t ind = s.find_last_of('.', std::string::npos); // last '.'
512 if (ind == std::string::npos) {
513 return false;
514 }
515 s.erase(ind);
516
517 // Now find the processor count...
518 ind = s.find_last_of('.', std::string::npos);
519 if (ind == std::string::npos) {
520 return false;
521 }
522
523 std::string tmp = s.substr(ind + 1); // Skip the '.'
524 processorCount_ = std::stoi(tmp);
525 if (processorCount_ <= 0) {
526 fmt::print(
527 stderr,
528 "\nERROR: (CPUP) Invalid processor count specified: '{}'. Must be greater than zero.\n",
529 processorCount_);
530 return false;
531 }
532 s.erase(ind);
533
534 // Should now be an extension...
535 ind = s.find_last_of('.', std::string::npos);
536 if (ind == std::string::npos) {
537 inExtension_ = "";
538 }
539 else {
540 inExtension_ = s.substr(ind + 1);
541 s.erase(ind);
542 }
543
544 // The directory path was stripped prior to entering this function,
545 // so remainder of 's' is just the new basename_
546 basename_ = s;
547 return true;
548 }
549
550 namespace {
551 std::string LowerCase(const std::string &name)
552 {
553 std::string s = name;
554 std::transform(s.begin(), s.end(), // source
555 s.begin(), // destination
556 ::tolower); // operation
557 return s;
558 }
559
560 void parse_variable_names(const char *tokens, Cpup::StringVector *variable_list)
561 {
562 // Break into tokens separated by ","
563 if (tokens != nullptr) {
564 std::string token_string(tokens);
565 Cpup::StringVector var_list = SLIB::tokenize(token_string, ",");
566 for (const auto &var : var_list) {
567 std::string low_var = LowerCase(var);
568 (*variable_list).push_back(low_var);
569 }
570 // Sort the list...
571 std::sort(variable_list->begin(), variable_list->end());
572 }
573 }
574
575 std::string find_matching_file(const std::string &path, const std::string &basename)
576 {
577 glob::glob g(basename + ".*.*");
578 #if !defined(__SUP_WINDOWS__)
579 struct dirent *entry = nullptr;
580 DIR *dp = nullptr;
581
582 dp = opendir(path.c_str());
583 if (dp != nullptr) {
584 while ((entry = readdir(dp))) {
585 std::string filename = entry->d_name;
586 if (glob::glob_match(filename, g)) {
587 closedir(dp);
588 return filename;
589 }
590 }
591 }
592 closedir(dp);
593 #endif
594 return "";
595 }
596 } // namespace