| 1 |
// Copyright(C) 2021, 2022, 2023 National Technology & Engineering Solutions |
| 2 |
// of Sandia, LLC (NTESS). Under the terms of Contract DE-NA0003525 with |
| 3 |
// NTESS, the U.S. Government retains certain rights in this software. |
| 4 |
// |
| 5 |
// See packages/seacas/LICENSE for details |
| 6 |
#include <exception> |
| 7 |
#include <fstream> |
| 8 |
#include <iterator> |
| 9 |
#include <string> |
| 10 |
|
| 11 |
#include "add_to_log.h" |
| 12 |
#include "fmt/chrono.h" |
| 13 |
#include "fmt/color.h" |
| 14 |
#include "fmt/ostream.h" |
| 15 |
#include "time_stamp.h" |
| 16 |
#include "tokenize.h" |
| 17 |
|
| 18 |
#include <exodusII.h> |
| 19 |
|
| 20 |
#include <Ionit_Initializer.h> |
| 21 |
#include <Ioss_MemoryUtils.h> |
| 22 |
#include <Ioss_ParallelUtils.h> |
| 23 |
#include <Ioss_SmartAssert.h> |
| 24 |
#include <Ioss_Utils.h> |
| 25 |
|
| 26 |
#include "Grid.h" |
| 27 |
#include "ZE_SystemInterface.h" |
| 28 |
|
| 29 |
#ifdef SEACAS_HAVE_MPI |
| 30 |
#include <mpi.h> |
| 31 |
#endif |
| 32 |
|
| 33 |
//! \file |
| 34 |
|
| 35 |
namespace { |
| 36 |
Grid define_lattice(SystemInterface &interFace, Ioss::ParallelUtils &pu); |
| 37 |
} // namespace |
| 38 |
|
| 39 |
std::string tsFormat = "[{:%H:%M:%S}]"; |
| 40 |
unsigned int debug_level = 0; |
| 41 |
|
| 42 |
template <typename INT> double zellij(SystemInterface &interFace, INT dummy); |
| 43 |
|
| 44 |
int main(int argc, char *argv[]) |
| 45 |
{ |
| 46 |
#ifdef SEACAS_HAVE_MPI |
| 47 |
MPI_Init(&argc, &argv); |
| 48 |
|
| 49 |
#endif |
| 50 |
|
| 51 |
Ioss::ParallelUtils pu{}; |
| 52 |
int my_rank = pu.parallel_rank(); |
| 53 |
|
| 54 |
try { |
| 55 |
if (my_rank == 0) { |
| 56 |
SystemInterface::show_version(); |
| 57 |
#ifdef SEACAS_HAVE_MPI |
| 58 |
fmt::print("\tParallel Capability Enabled.\n"); |
| 59 |
#else |
| 60 |
fmt::print("\tParallel Capability Not Enabled.\n"); |
| 61 |
#endif |
| 62 |
} |
| 63 |
Ioss::Init::Initializer io; |
| 64 |
|
| 65 |
SystemInterface interFace(my_rank); |
| 66 |
bool ok = interFace.parse_options(argc, argv); |
| 67 |
|
| 68 |
if (!ok) { |
| 69 |
fmt::print(stderr, fmt::fg(fmt::color::red), |
| 70 |
"\nERROR: Problems parsing command line arguments.\n\n"); |
| 71 |
exit(EXIT_FAILURE); |
| 72 |
} |
| 73 |
|
| 74 |
debug_level = interFace.debug(); |
| 75 |
|
| 76 |
if (debug_level & 1) { |
| 77 |
ex_opts(EX_VERBOSE | EX_DEBUG); |
| 78 |
} |
| 79 |
else { |
| 80 |
ex_opts(0); |
| 81 |
} |
| 82 |
|
| 83 |
double time = 0.0; |
| 84 |
if (interFace.ints32bit()) { |
| 85 |
time = zellij(interFace, 0); |
| 86 |
} |
| 87 |
else { |
| 88 |
time = zellij(interFace, static_cast<int64_t>(0)); |
| 89 |
} |
| 90 |
|
| 91 |
if (my_rank == 0) { |
| 92 |
fmt::print("\n Zellij execution successful.\n"); |
| 93 |
add_to_log(argv[0], time); |
| 94 |
} |
| 95 |
|
| 96 |
#ifdef SEACAS_HAVE_MPI |
| 97 |
MPI_Finalize(); |
| 98 |
#endif |
| 99 |
} |
| 100 |
catch (std::exception &e) { |
| 101 |
fmt::print(stderr, fmt::fg(fmt::color::red), "ERROR: Standard exception: {}\n", e.what()); |
| 102 |
} |
| 103 |
} |
| 104 |
|
| 105 |
template <typename INT> double zellij(SystemInterface &interFace, INT /*dummy*/) |
| 106 |
{ |
| 107 |
double begin = Ioss::Utils::timer(); |
| 108 |
Ioss::ParallelUtils pu{}; |
| 109 |
|
| 110 |
if (debug_level & 2) { |
| 111 |
fmt::print(stderr, "{} Begin Execution\n", time_stamp(tsFormat)); |
| 112 |
} |
| 113 |
|
| 114 |
auto grid = define_lattice(interFace, pu); |
| 115 |
|
| 116 |
grid.generate_sidesets(); |
| 117 |
grid.set_coordinate_offsets(); |
| 118 |
grid.decompose(interFace.decomp_method()); |
| 119 |
|
| 120 |
if (debug_level & 2) { |
| 121 |
fmt::print(stderr, "{} Lattice Decomposed\n", time_stamp(tsFormat)); |
| 122 |
} |
| 123 |
|
| 124 |
// All unit cells have been mapped into the IxJ grid, now calculate all node / element offsets |
| 125 |
// and the global node and element counts... |
| 126 |
// |
| 127 |
// Iterate through the grid starting with (0,0) and accumulate node and element counts... |
| 128 |
grid.process(interFace, (INT)0); |
| 129 |
|
| 130 |
/*************************************************************************/ |
| 131 |
// EXIT program |
| 132 |
if (debug_level & 2) { |
| 133 |
fmt::print(stderr, "{} Execution Complete\n", time_stamp(tsFormat)); |
| 134 |
} |
| 135 |
|
| 136 |
double end = Ioss::Utils::timer(); |
| 137 |
double hwm = (double)Ioss::MemoryUtils::get_hwm_memory_info() / 1024.0 / 1024.0; |
| 138 |
if (pu.parallel_rank() == 0) { |
| 139 |
fmt::print("\n Total Execution Time = {:.5} seconds.\n", end - begin); |
| 140 |
fmt::print(" High-Water Memory Use = {:.3} MiBytes.\n", hwm); |
| 141 |
} |
| 142 |
return (end - begin); |
| 143 |
} |
| 144 |
|
| 145 |
namespace { |
| 146 |
Grid define_lattice(SystemInterface &interFace, Ioss::ParallelUtils &pu) |
| 147 |
{ |
| 148 |
int my_rank = pu.parallel_rank(); |
| 149 |
|
| 150 |
Grid grid(interFace); |
| 151 |
|
| 152 |
if (debug_level & 2) { |
| 153 |
pu.progress("Defining Unit Cells..."); |
| 154 |
} |
| 155 |
std::string filename = interFace.lattice(); |
| 156 |
|
| 157 |
std::ifstream input(filename, std::ios::in); |
| 158 |
if (!input.is_open()) { |
| 159 |
fmt::print(stderr, fmt::fg(fmt::color::red), |
| 160 |
"\nERROR: Could not open/access lattice file '{}'\n", filename); |
| 161 |
exit(EXIT_FAILURE); |
| 162 |
} |
| 163 |
|
| 164 |
bool in_dictionary{false}; |
| 165 |
bool in_lattice{false}; |
| 166 |
std::string line; |
| 167 |
while (getline(input, line)) { |
| 168 |
if (line.empty()) { |
| 169 |
continue; |
| 170 |
} |
| 171 |
|
| 172 |
auto tokens = Ioss::tokenize(line, " "); |
| 173 |
if (tokens[0] == "BEGIN_DICTIONARY") { |
| 174 |
SMART_ASSERT(!in_lattice && !in_dictionary); |
| 175 |
in_dictionary = true; |
| 176 |
} |
| 177 |
else if (tokens[0] == "END_DICTIONARY") { |
| 178 |
SMART_ASSERT(!in_lattice && in_dictionary); |
| 179 |
in_dictionary = false; |
| 180 |
if (debug_level & 2) { |
| 181 |
pu.progress("Unit Cells Defined..."); |
| 182 |
} |
| 183 |
} |
| 184 |
else if (in_dictionary) { |
| 185 |
if (tokens.size() != 2) { |
| 186 |
fmt::print(stderr, fmt::fg(fmt::color::red), |
| 187 |
"\nERROR: There are {} entries on a lattice dictionary line; there should be " |
| 188 |
"only 2:\n\t'{}'.\n\n", |
| 189 |
tokens.size(), line); |
| 190 |
exit(EXIT_FAILURE); |
| 191 |
} |
| 192 |
|
| 193 |
grid.add_unit_cell(tokens[0], tokens[1], interFace.ints32bit()); |
| 194 |
} |
| 195 |
else if (tokens[0] == "BEGIN_LATTICE") { |
| 196 |
SMART_ASSERT(!in_lattice && !in_dictionary); |
| 197 |
in_lattice = true; |
| 198 |
break; |
| 199 |
} |
| 200 |
} |
| 201 |
|
| 202 |
if (!in_lattice) { |
| 203 |
fmt::print( |
| 204 |
stderr, fmt::fg(fmt::color::red), |
| 205 |
"\nERROR: Reached end of input file without finding a 'BEGIN_LATTICE' command\n\n"); |
| 206 |
exit(EXIT_FAILURE); |
| 207 |
} |
| 208 |
|
| 209 |
// Tokenize line to get I J K size of lattice |
| 210 |
auto tokens = Ioss::tokenize(line, " "); |
| 211 |
SMART_ASSERT(tokens[0] == "BEGIN_LATTICE")(tokens[0])(line); |
| 212 |
|
| 213 |
if (tokens.size() != 4) { |
| 214 |
fmt::print(stderr, fmt::fg(fmt::color::red), |
| 215 |
"\nERROR: The 'BEGIN_LATTICE' line has incorrect syntax. It should be " |
| 216 |
"'BEGIN_LATTICE I J K'\n" |
| 217 |
"\tThe line was '{}'\n\n", |
| 218 |
line); |
| 219 |
exit(EXIT_FAILURE); |
| 220 |
} |
| 221 |
|
| 222 |
SMART_ASSERT(tokens.size() == 4)(tokens.size())(line); |
| 223 |
int II = std::stoi(tokens[1]); |
| 224 |
int JJ = std::stoi(tokens[2]); |
| 225 |
int KK = std::stoi(tokens[3]); |
| 226 |
SMART_ASSERT(KK == 1); |
| 227 |
|
| 228 |
if (interFace.repeat() > 1) { |
| 229 |
II *= interFace.repeat(); |
| 230 |
JJ *= interFace.repeat(); |
| 231 |
} |
| 232 |
|
| 233 |
grid.set_extent(II, JJ, KK); |
| 234 |
grid.handle_file_count(); |
| 235 |
|
| 236 |
if (my_rank == 0) { |
| 237 |
fmt::print("\n Lattice:\tUnit Cells: {},\tGrid Size: {} x {} x {}\n", |
| 238 |
fmt::group_digits(grid.unit_cells().size()), fmt::group_digits(II), |
| 239 |
fmt::group_digits(JJ), fmt::group_digits(KK)); |
| 240 |
} |
| 241 |
if (interFace.ranks() > 1) { |
| 242 |
fmt::print(" \t[{}] Ranks: {}, Outputting {} ranks starting at rank {}.\n", my_rank, |
| 243 |
fmt::group_digits(interFace.ranks()), fmt::group_digits(interFace.rank_count()), |
| 244 |
fmt::group_digits(interFace.start_rank())); |
| 245 |
} |
| 246 |
|
| 247 |
// Now process the lattice portion of the lattice file... |
| 248 |
size_t row{0}; |
| 249 |
while (getline(input, line)) { |
| 250 |
if (line.empty()) { |
| 251 |
continue; |
| 252 |
} |
| 253 |
|
| 254 |
tokens = Ioss::tokenize(line, " "); |
| 255 |
if (tokens[0] == "END_LATTICE") { |
| 256 |
SMART_ASSERT(in_lattice && !in_dictionary); |
| 257 |
in_lattice = false; |
| 258 |
|
| 259 |
// Check row count to make sure matches 'J' size of lattice |
| 260 |
if (row != grid.JJ()) { |
| 261 |
fmt::print(stderr, fmt::fg(fmt::color::red), |
| 262 |
"\nERROR: Only {} rows of the {} x {} lattice were defined.\n\n", row, II, JJ); |
| 263 |
exit(EXIT_FAILURE); |
| 264 |
} |
| 265 |
break; |
| 266 |
} |
| 267 |
if (in_lattice) { |
| 268 |
// TODO: Currently assumes that each row in the lattice is defined on a single row; |
| 269 |
// This will need to be relaxed since a lattice of 5000x5000 would result in |
| 270 |
// lines that are too long and would be easier to split a row over multiple lines... |
| 271 |
if (tokens.size() * interFace.repeat() != grid.II()) { |
| 272 |
fmt::print( |
| 273 |
stderr, fmt::fg(fmt::color::red), |
| 274 |
"\nERROR: Line {} of the lattice definition has {} entries. It should have {}.\n\n", |
| 275 |
row + 1, tokens.size() * interFace.repeat(), grid.II()); |
| 276 |
exit(EXIT_FAILURE); |
| 277 |
} |
| 278 |
|
| 279 |
if (row >= grid.JJ()) { |
| 280 |
fmt::print(stderr, fmt::fg(fmt::color::red), |
| 281 |
"\nERROR: There are too many rows in the lattice definition. The lattice is " |
| 282 |
"{} x {}.\n\n", |
| 283 |
grid.II(), grid.JJ()); |
| 284 |
exit(EXIT_FAILURE); |
| 285 |
} |
| 286 |
|
| 287 |
if (tokens.size() * interFace.repeat() != grid.II()) { |
| 288 |
fmt::print(stderr, fmt::fg(fmt::color::red), |
| 289 |
"\nERROR: In row {}, there is an incorrect number of entries. There should " |
| 290 |
"be {}, but found {}.\n", |
| 291 |
row + 1, grid.II(), tokens.size() * interFace.repeat()); |
| 292 |
exit(EXIT_FAILURE); |
| 293 |
} |
| 294 |
|
| 295 |
auto repeat = interFace.repeat(); |
| 296 |
for (int j = 0; j < repeat; j++) { |
| 297 |
size_t col = 0; |
| 298 |
for (auto &key : tokens) { |
| 299 |
|
| 300 |
for (int i = 0; i < repeat; i++) { |
| 301 |
if (!grid.initialize(col++, row, key)) { |
| 302 |
fmt::print( |
| 303 |
stderr, fmt::fg(fmt::color::red), |
| 304 |
"\nERROR: In row {}, column {}, the lattice specifies a unit cell ({}) that " |
| 305 |
"has not been defined.\n\n", |
| 306 |
row + 1, col + 1, key); |
| 307 |
exit(EXIT_FAILURE); |
| 308 |
} |
| 309 |
} |
| 310 |
} |
| 311 |
row++; |
| 312 |
} |
| 313 |
} |
| 314 |
} |
| 315 |
if (debug_level & 2) { |
| 316 |
fmt::print(stderr, "{} Lattice Defined\n", time_stamp(tsFormat)); |
| 317 |
} |
| 318 |
return grid; |
| 319 |
} |
| 320 |
} // namespace |