| 1 |
// Copyright(C) 2021, 2022, 2023, 2024 National Technology & Engineering Solutions |
| 2 |
// of Sandia, LLC (NTESS). Under the terms of Contract DE-NA0003525 with |
| 3 |
// NTESS, the U.S. Government retains certain rights in this software. |
| 4 |
// |
| 5 |
// See packages/seacas/LICENSE for details |
| 6 |
|
| 7 |
#include <cstdlib> |
| 8 |
#include <numeric> |
| 9 |
|
| 10 |
#include "Cell.h" |
| 11 |
#include "Decompose.h" |
| 12 |
#include "Grid.h" |
| 13 |
#include "ZE_SystemInterface.h" |
| 14 |
#include "ZE_Version.h" |
| 15 |
#include <Ioss_CommSet.h> |
| 16 |
#include <Ioss_ElementBlock.h> |
| 17 |
#include <Ioss_IOFactory.h> |
| 18 |
#include <Ioss_NodeBlock.h> |
| 19 |
#include <Ioss_ParallelUtils.h> |
| 20 |
#include <Ioss_Region.h> |
| 21 |
#include <Ioss_SideBlock.h> |
| 22 |
#include <Ioss_SideSet.h> |
| 23 |
#include <Ioss_SmartAssert.h> |
| 24 |
|
| 25 |
#include <exodusII.h> |
| 26 |
#include <fmt/chrono.h> |
| 27 |
#include <fmt/color.h> |
| 28 |
#include <fmt/ostream.h> |
| 29 |
#include <fmt/ranges.h> |
| 30 |
#include <open_file_limit.h> |
| 31 |
#include <time_stamp.h> |
| 32 |
#include <tokenize.h> |
| 33 |
|
| 34 |
//! \file |
| 35 |
|
| 36 |
extern unsigned int debug_level; |
| 37 |
|
| 38 |
#if __cplusplus == 201103L |
| 39 |
// From: https://stackoverflow.com/questions/17902405/how-to-implement-make-unique-function-in-c11 |
| 40 |
namespace std { |
| 41 |
template <class T> struct _Unique_if |
| 42 |
{ |
| 43 |
typedef unique_ptr<T> _Single_object; |
| 44 |
}; |
| 45 |
|
| 46 |
template <class T> struct _Unique_if<T[]> |
| 47 |
{ |
| 48 |
typedef unique_ptr<T[]> _Unknown_bound; |
| 49 |
}; |
| 50 |
|
| 51 |
template <class T, size_t N> struct _Unique_if<T[N]> |
| 52 |
{ |
| 53 |
typedef void _Known_bound; |
| 54 |
}; |
| 55 |
|
| 56 |
template <class T, class... Args> |
| 57 |
typename _Unique_if<T>::_Single_object make_unique(Args &&...args) |
| 58 |
{ |
| 59 |
return unique_ptr<T>(new T(std::forward<Args>(args)...)); |
| 60 |
} |
| 61 |
|
| 62 |
template <class T> typename _Unique_if<T>::_Unknown_bound make_unique(size_t n) |
| 63 |
{ |
| 64 |
typedef typename remove_extent<T>::type U; |
| 65 |
return unique_ptr<T>(new U[n]()); |
| 66 |
} |
| 67 |
|
| 68 |
template <class T, class... Args> |
| 69 |
typename _Unique_if<T>::_Known_bound make_unique(Args &&...) = delete; |
| 70 |
} // namespace std |
| 71 |
#endif |
| 72 |
|
| 73 |
namespace { |
| 74 |
std::string tsFormat = "[{:%H:%M:%S}]"; |
| 75 |
int axis_index(const std::string &axis_str) |
| 76 |
{ |
| 77 |
char axis = axis_str[0]; |
| 78 |
if (axis == 'x' || axis == 'i') { |
| 79 |
return 0; |
| 80 |
} |
| 81 |
if (axis == 'X' || axis == 'I') { |
| 82 |
return 1; |
| 83 |
} |
| 84 |
if (axis == 'y' || axis == 'j') { |
| 85 |
return 2; |
| 86 |
} |
| 87 |
if (axis == 'Y' || axis == 'J') { |
| 88 |
return 3; |
| 89 |
} |
| 90 |
if (axis == 'z' || axis == 'k') { |
| 91 |
return 4; |
| 92 |
} |
| 93 |
if (axis == 'Z' || axis == 'K') { |
| 94 |
return 5; |
| 95 |
} |
| 96 |
return -1; |
| 97 |
} |
| 98 |
|
| 99 |
unsigned int which_sidesets(const std::string &sideset_string) |
| 100 |
{ |
| 101 |
unsigned generate = 0; |
| 102 |
for (const auto &c : sideset_string) { |
| 103 |
if (c == 'x' || c == 'i') { |
| 104 |
generate |= (unsigned)Flg::MIN_I; |
| 105 |
} |
| 106 |
if (c == 'y' || c == 'j') { |
| 107 |
generate |= (unsigned)Flg::MIN_J; |
| 108 |
} |
| 109 |
if (c == 'z' || c == 'k') { |
| 110 |
generate |= (unsigned)Flg::MIN_K; |
| 111 |
} |
| 112 |
if (c == 'X' || c == 'I') { |
| 113 |
generate |= (unsigned)Flg::MAX_I; |
| 114 |
} |
| 115 |
if (c == 'Y' || c == 'J') { |
| 116 |
generate |= (unsigned)Flg::MAX_J; |
| 117 |
} |
| 118 |
if (c == 'Z' || c == 'K') { |
| 119 |
generate |= (unsigned)Flg::MAX_K; |
| 120 |
} |
| 121 |
} |
| 122 |
return generate; |
| 123 |
} |
| 124 |
|
| 125 |
Ioss::PropertyManager parse_properties(SystemInterface &interFace, int int_size); |
| 126 |
|
| 127 |
std::shared_ptr<Ioss::Region> create_input_region(const std::string &key, std::string filename, |
| 128 |
bool ints_32_bits); |
| 129 |
|
| 130 |
template <typename INT> |
| 131 |
std::vector<INT> generate_node_map(Grid &grid, const Cell &cell, Mode mode, INT /*dummy*/); |
| 132 |
|
| 133 |
void output_summary(Ioss::Region *region, std::ostream &strm); |
| 134 |
size_t handle_elements(Grid &grid, int start_rank, int rank_count); |
| 135 |
size_t handle_nodes(Grid &grid, int start_rank, int rank_count); |
| 136 |
void handle_communications(Grid &grid, int start_rank, int rank_count); |
| 137 |
void handle_surfaces(Grid &grid, int start_rank, int rank_count); |
| 138 |
void generate_surfaces(Grid &grid, int start_rank, int rank_count); |
| 139 |
} // namespace |
| 140 |
|
| 141 |
Grid::Grid(SystemInterface &interFace) |
| 142 |
: m_offset(interFace.offset()), m_scaleFactor(interFace.scale_factor()), |
| 143 |
m_parallelSize(interFace.ranks()), m_rankCount(interFace.rank_count()), |
| 144 |
m_startRank(interFace.start_rank()), m_equivalenceNodes(interFace.equivalence_nodes()), |
| 145 |
m_useInternalSidesets(!interFace.ignore_internal_sidesets()), |
| 146 |
m_subCycle(interFace.subcycle()), m_minimizeOpenFiles(interFace.minimize_open_files()), |
| 147 |
m_generatedSideSets(which_sidesets(interFace.sideset_surfaces())) |
| 148 |
{ |
| 149 |
set_sideset_names(interFace.sideset_names()); |
| 150 |
} |
| 151 |
|
| 152 |
void Grid::set_extent(size_t extent_i, size_t extent_j, size_t /* unused */) |
| 153 |
{ |
| 154 |
m_gridI = extent_i; |
| 155 |
m_gridJ = extent_j; |
| 156 |
m_grid.resize(m_gridI * m_gridJ); |
| 157 |
} |
| 158 |
|
| 159 |
bool Grid::initialize(size_t i, size_t j, const std::string &key) |
| 160 |
{ |
| 161 |
if (unit_cells().find(key) == unit_cells().end()) { |
| 162 |
return false; |
| 163 |
} |
| 164 |
const auto &unit_cell = unit_cells()[key]; |
| 165 |
SMART_ASSERT(unit_cell->m_region != nullptr)(i)(j)(key); |
| 166 |
|
| 167 |
auto &cell = get_cell(i, j); |
| 168 |
cell.initialize(i, j, unit_cell); |
| 169 |
return true; |
| 170 |
} |
| 171 |
|
| 172 |
void Grid::add_unit_cell(const std::string &key, const std::string &unit_filename, bool ints32bit) |
| 173 |
{ |
| 174 |
static size_t open_files = open_file_limit(); |
| 175 |
if (!minimize_open_files(Minimize::UNIT) && unit_cells().size() >= open_files) { |
| 176 |
// Just hit the limit... Close all previous unit_cell files and set the minimize_open_files |
| 177 |
// behavior to UNIT. |
| 178 |
for (const auto &unit_cell : m_unitCells) { |
| 179 |
unit_cell.second->m_region->get_database()->closeDatabase(); |
| 180 |
} |
| 181 |
fmt::print(stderr, fmt::fg(fmt::color::yellow), |
| 182 |
"\nWARNING: Number of unit cells exceeds open file limit. Setting " |
| 183 |
"'mimimize_open_file' mode to UNIT.\n\n"); |
| 184 |
m_minimizeOpenFiles = Minimize((unsigned)m_minimizeOpenFiles | (unsigned)Minimize::UNIT); |
| 185 |
} |
| 186 |
|
| 187 |
if (unit_cells().find(key) != unit_cells().end()) { |
| 188 |
fmt::print(stderr, fmt::fg(fmt::color::red), |
| 189 |
"\nERROR: There is a duplicate `unit cell` ({}) in the lattice dictionary.\n\n", |
| 190 |
key); |
| 191 |
exit(EXIT_FAILURE); |
| 192 |
} |
| 193 |
|
| 194 |
auto region = create_input_region(key, unit_filename, ints32bit); |
| 195 |
|
| 196 |
if (region == nullptr) { |
| 197 |
fmt::print(stderr, fmt::fg(fmt::color::red), |
| 198 |
"\nERROR: Unable to open the database '{}' associated with the unit cell '{}'.\n\n", |
| 199 |
unit_filename, key); |
| 200 |
exit(EXIT_FAILURE); |
| 201 |
} |
| 202 |
|
| 203 |
unit_cells().emplace(key, std::make_shared<UnitCell>(region)); |
| 204 |
if (minimize_open_files(Minimize::UNIT)) { |
| 205 |
unit_cells()[key]->m_region->get_database()->closeDatabase(); |
| 206 |
} |
| 207 |
|
| 208 |
if (debug_level & 2) { |
| 209 |
m_pu.progress(fmt::format("\tCreated Unit Cell {}", key)); |
| 210 |
} |
| 211 |
} |
| 212 |
|
| 213 |
void Grid::set_coordinate_offsets() |
| 214 |
{ |
| 215 |
// All unit_cells have same X, Y (and Z) extent. Only need X and Y |
| 216 |
auto x_range = get_cell(0, 0).get_coordinate_range(Axis::X); |
| 217 |
auto y_range = get_cell(0, 0).get_coordinate_range(Axis::Y); |
| 218 |
auto delta_x = x_range.second - x_range.first; |
| 219 |
auto delta_y = y_range.second - y_range.first; |
| 220 |
|
| 221 |
for (size_t j = 0; j < JJ(); j++) { |
| 222 |
for (size_t i = 0; i < II(); i++) { |
| 223 |
auto &cell = get_cell(i, j); |
| 224 |
cell.m_offX = delta_x * (double)i; |
| 225 |
cell.m_offY = delta_y * (double)j; |
| 226 |
|
| 227 |
if (debug_level & 2) { |
| 228 |
util().progress( |
| 229 |
fmt::format("\tCell({}, {}) X = {}, Y = {}", i, j, cell.m_offX, cell.m_offY)); |
| 230 |
} |
| 231 |
} |
| 232 |
} |
| 233 |
} |
| 234 |
|
| 235 |
void Grid::create_output_regions(SystemInterface &interFace) |
| 236 |
{ |
| 237 |
if (debug_level & 2) { |
| 238 |
util().progress(__func__); |
| 239 |
} |
| 240 |
m_outputRegions.resize(interFace.ranks()); |
| 241 |
|
| 242 |
int int_size = interFace.ints32bit() ? 4 : 8; |
| 243 |
Ioss::PropertyManager properties = parse_properties(interFace, int_size); |
| 244 |
if (parallel_size() == 1) { |
| 245 |
properties.add(Ioss::Property("OMIT_EXODUS_NUM_MAPS", 1)); |
| 246 |
} |
| 247 |
// Disable this for now. Readers need to be modified and propagated to allow this. |
| 248 |
// properties.add(Ioss::Property("MINIMAL_NEMESIS_DATA", 1)); |
| 249 |
|
| 250 |
if (debug_level & 2) { |
| 251 |
properties.add(Ioss::Property("ENABLE_TRACING", 1)); |
| 252 |
} |
| 253 |
|
| 254 |
for (int i = m_startRank; i < m_startRank + m_rankCount; i++) { |
| 255 |
// Define the output database(s)... |
| 256 |
if (m_parallelSize > 1) { |
| 257 |
properties.add(Ioss::Property("processor_count", parallel_size())); |
| 258 |
properties.add(Ioss::Property("my_processor", i)); |
| 259 |
} |
| 260 |
Ioss::DatabaseIO *dbo = |
| 261 |
Ioss::IOFactory::create("exodus", interFace.outputName_, Ioss::WRITE_RESTART, |
| 262 |
Ioss::ParallelUtils::comm_self(), properties); |
| 263 |
if (dbo == nullptr || !dbo->ok(true)) { |
| 264 |
std::exit(EXIT_FAILURE); |
| 265 |
} |
| 266 |
m_outputRegions[i] = std::make_unique<Ioss::Region>(dbo, "zellij_output_region"); |
| 267 |
output_region(i)->begin_mode(Ioss::STATE_DEFINE_MODEL); |
| 268 |
output_region(i)->property_add(Ioss::Property("code_name", qainfo[0])); |
| 269 |
output_region(i)->property_add(Ioss::Property("code_version", qainfo[2])); |
| 270 |
|
| 271 |
if (interFace.minimize_open_files() == Minimize::OUTPUT || |
| 272 |
interFace.minimize_open_files() == Minimize::ALL) { |
| 273 |
output_region(i)->get_database()->closeDatabase(); |
| 274 |
} |
| 275 |
} |
| 276 |
} |
| 277 |
|
| 278 |
void Grid::set_sideset_names(const std::string &names) |
| 279 |
{ |
| 280 |
// 'names' is a string of the form "Axis:name,Axis:name, ..." |
| 281 |
// For example "x:left,y:top,X:right,Y:bottom" |
| 282 |
// Parse the list and only update the names specified in the list... |
| 283 |
|
| 284 |
if (names.empty()) { |
| 285 |
return; |
| 286 |
} |
| 287 |
|
| 288 |
auto tokens = Ioss::tokenize(names, ","); |
| 289 |
for (auto &token : tokens) { |
| 290 |
auto axis = token.substr(0, 1); |
| 291 |
bool valid = axis.find_first_not_of("ijkIJKxyzXYZ") == std::string::npos; |
| 292 |
if (!valid) { |
| 293 |
fmt::print(stderr, fmt::fg(fmt::color::red), |
| 294 |
"\nERROR: Invalid axis '{}' specified for sideset name. Valid is one of " |
| 295 |
"'ijkIJKxyzXYZ'.\n\n", |
| 296 |
axis); |
| 297 |
exit(EXIT_FAILURE); |
| 298 |
} |
| 299 |
|
| 300 |
// Now get the name... |
| 301 |
auto ss_name = token.substr(2); |
| 302 |
|
| 303 |
// Update the name in the list of generated sideset names... |
| 304 |
auto index = axis_index(axis); |
| 305 |
SMART_ASSERT(index >= 0)(axis)(index); |
| 306 |
generated_surface_names[index] = std::move(ss_name); |
| 307 |
} |
| 308 |
} |
| 309 |
|
| 310 |
void Grid::decompose(const std::string &method) |
| 311 |
{ |
| 312 |
if (debug_level & 2) { |
| 313 |
util().progress(__func__); |
| 314 |
} |
| 315 |
decompose_grid(*this, m_parallelSize, method); |
| 316 |
|
| 317 |
categorize_processor_boundaries(); |
| 318 |
} |
| 319 |
|
| 320 |
void Grid::categorize_processor_boundaries() |
| 321 |
{ |
| 322 |
if (debug_level & 2) { |
| 323 |
util().progress(__func__); |
| 324 |
} |
| 325 |
|
| 326 |
// Now iterate the cells and tell each cell the rank of all neighboring cells... |
| 327 |
// boundary with its "left" or "lower" neighboring cell. |
| 328 |
|
| 329 |
for (size_t j = 0; j < JJ(); j++) { |
| 330 |
for (size_t i = 0; i < II(); i++) { |
| 331 |
auto &cell = get_cell(i, j); |
| 332 |
if (i > 0) { |
| 333 |
const auto &left = get_cell(i - 1, j); |
| 334 |
cell.set_rank(Loc::L, left.rank(Loc::C)); |
| 335 |
if (j > 0) { |
| 336 |
const auto &BL = get_cell(i - 1, j - 1); |
| 337 |
cell.set_rank(Loc::BL, BL.rank(Loc::C)); |
| 338 |
} |
| 339 |
if (j < JJ() - 1) { |
| 340 |
const auto &TL = get_cell(i - 1, j + 1); |
| 341 |
cell.set_rank(Loc::TL, TL.rank(Loc::C)); |
| 342 |
} |
| 343 |
} |
| 344 |
if (i < II() - 1) { |
| 345 |
const auto &right = get_cell(i + 1, j); |
| 346 |
cell.set_rank(Loc::R, right.rank(Loc::C)); |
| 347 |
if (j > 0) { |
| 348 |
const auto &BR = get_cell(i + 1, j - 1); |
| 349 |
cell.set_rank(Loc::BR, BR.rank(Loc::C)); |
| 350 |
} |
| 351 |
if (j < JJ() - 1) { |
| 352 |
const auto &TR = get_cell(i + 1, j + 1); |
| 353 |
cell.set_rank(Loc::TR, TR.rank(Loc::C)); |
| 354 |
} |
| 355 |
} |
| 356 |
if (j > 0) { |
| 357 |
const auto &B = get_cell(i, j - 1); |
| 358 |
cell.set_rank(Loc::B, B.rank(Loc::C)); |
| 359 |
} |
| 360 |
if (j < JJ() - 1) { |
| 361 |
const auto &T = get_cell(i, j + 1); |
| 362 |
cell.set_rank(Loc::T, T.rank(Loc::C)); |
| 363 |
} |
| 364 |
} |
| 365 |
} |
| 366 |
|
| 367 |
if (debug_level & 32) { |
| 368 |
auto width = Ioss::Utils::number_width(parallel_size()); |
| 369 |
for (size_t j = 0; j < JJ(); j++) { |
| 370 |
for (size_t i = 0; i < II(); i++) { |
| 371 |
const auto &cell = get_cell(i, j); |
| 372 |
auto left = cell.processor_boundary(Loc::L) ? '<' : ' '; |
| 373 |
auto below = cell.processor_boundary(Loc::B) ? '^' : ' '; |
| 374 |
fmt::print(" {0}{1:{3}}{2}", left, cell.rank(Loc::C), below, width); |
| 375 |
} |
| 376 |
fmt::print("\n"); |
| 377 |
} |
| 378 |
} |
| 379 |
} |
| 380 |
|
| 381 |
void Grid::generate_sidesets() |
| 382 |
{ |
| 383 |
if (m_generatedSideSets != 0) { |
| 384 |
for (const auto &unit_cell : m_unitCells) { |
| 385 |
unit_cell.second->generate_boundary_faces(m_generatedSideSets); |
| 386 |
} |
| 387 |
} |
| 388 |
} |
| 389 |
|
| 390 |
void Grid::internal_process() |
| 391 |
{ |
| 392 |
if (debug_level & 2) { |
| 393 |
util().progress(__func__); |
| 394 |
} |
| 395 |
auto node_count = handle_nodes(*this, m_startRank, m_rankCount); |
| 396 |
auto element_count = handle_elements(*this, m_startRank, m_rankCount); |
| 397 |
handle_communications(*this, m_startRank, m_rankCount); |
| 398 |
if (m_useInternalSidesets) { |
| 399 |
handle_surfaces(*this, m_startRank, m_rankCount); |
| 400 |
} |
| 401 |
generate_surfaces(*this, m_startRank, m_rankCount); |
| 402 |
|
| 403 |
for (int i = m_startRank; i < m_startRank + m_rankCount; i++) { |
| 404 |
output_region(i)->end_mode(Ioss::STATE_DEFINE_MODEL); |
| 405 |
if (debug_level & 64) { |
| 406 |
output_summary(output_region(i), std::cerr); |
| 407 |
} |
| 408 |
} |
| 409 |
if (util().parallel_rank() == 0) { |
| 410 |
fmt::print(" {} Nodes; {} Elements.\n", fmt::group_digits(node_count), |
| 411 |
fmt::group_digits(element_count)); |
| 412 |
} |
| 413 |
} |
| 414 |
|
| 415 |
template void Grid::process(SystemInterface &, int64_t); |
| 416 |
template void Grid::process(SystemInterface &, int); |
| 417 |
|
| 418 |
template <typename INT> void Grid::process(SystemInterface &interFace, INT /* dummy */) |
| 419 |
{ |
| 420 |
bool subcycle = m_subCycle; |
| 421 |
int start_rank = m_startRank; |
| 422 |
int rank_count = m_rankCount; |
| 423 |
int end_rank = subcycle ? m_parallelSize : m_startRank + m_rankCount; |
| 424 |
|
| 425 |
if (end_rank > m_parallelSize) { |
| 426 |
end_rank = m_parallelSize; |
| 427 |
m_rankCount = m_parallelSize - m_startRank; |
| 428 |
} |
| 429 |
|
| 430 |
for (int begin = start_rank; begin < end_rank; begin += rank_count) { |
| 431 |
m_startRank = begin; |
| 432 |
if (m_startRank + m_rankCount > m_parallelSize) { |
| 433 |
m_rankCount = m_parallelSize - m_startRank; |
| 434 |
} |
| 435 |
if (debug_level & 2) { |
| 436 |
fmt::print(stderr, "{} Processing Ranks {} to {}\n", time_stamp(tsFormat), begin, |
| 437 |
begin + rank_count - 1); |
| 438 |
} |
| 439 |
|
| 440 |
create_output_regions(interFace); |
| 441 |
|
| 442 |
internal_process(); |
| 443 |
if (debug_level & 2) { |
| 444 |
fmt::print(stderr, "{} Lattice Processing Finalized\n", time_stamp(tsFormat)); |
| 445 |
} |
| 446 |
|
| 447 |
output_model(INT(0)); |
| 448 |
if (debug_level & 2) { |
| 449 |
fmt::print(stderr, "{} Model Output\n", time_stamp(tsFormat)); |
| 450 |
} |
| 451 |
} |
| 452 |
} |
| 453 |
|
| 454 |
template void Grid::output_model(int64_t); |
| 455 |
template void Grid::output_model(int); |
| 456 |
|
| 457 |
// Write the output database(s) for all ranks... |
| 458 |
template <typename INT> void Grid::output_model(INT /*dummy*/) |
| 459 |
{ |
| 460 |
if (debug_level & 2) { |
| 461 |
util().progress(__func__); |
| 462 |
} |
| 463 |
// Coordinates do not depend on order of operations, so can do these |
| 464 |
// a rank at a time which can be more efficient once we exceed |
| 465 |
// maximum number of open files... Also easier to parallelize... |
| 466 |
for (int r = m_startRank; r < m_startRank + m_rankCount; r++) { |
| 467 |
for (size_t j = 0; j < JJ(); j++) { |
| 468 |
for (size_t i = 0; i < II(); i++) { |
| 469 |
const auto &cell = get_cell(i, j); |
| 470 |
if (cell.rank(Loc::C) == r) { |
| 471 |
output_nodal_coordinates(cell); |
| 472 |
} |
| 473 |
} |
| 474 |
} |
| 475 |
if (minimize_open_files(Minimize::OUTPUT)) { |
| 476 |
output_region(r)->get_database()->closeDatabase(); |
| 477 |
} |
| 478 |
} |
| 479 |
if (debug_level & 2) { |
| 480 |
util().progress("\tEnd Nodal Coordinate Output"); |
| 481 |
} |
| 482 |
|
| 483 |
// Surfaces do not depend on order of operations, so can do these |
| 484 |
// a rank at a time which can be more efficient once we exceed |
| 485 |
// maximum number of open files... Also easier to parallelize... |
| 486 |
for (int r = m_startRank; r < m_startRank + m_rankCount; r++) { |
| 487 |
for (size_t j = 0; j < JJ(); j++) { |
| 488 |
for (size_t i = 0; i < II(); i++) { |
| 489 |
auto &cell = get_cell(i, j); |
| 490 |
if (cell.rank(Loc::C) == r) { |
| 491 |
if (m_useInternalSidesets) { |
| 492 |
output_surfaces(cell, INT(0)); |
| 493 |
} |
| 494 |
output_generated_surfaces(cell, INT(0)); |
| 495 |
} |
| 496 |
} |
| 497 |
} |
| 498 |
if (minimize_open_files(Minimize::OUTPUT)) { |
| 499 |
output_region(r)->get_database()->closeDatabase(); |
| 500 |
} |
| 501 |
} |
| 502 |
if (debug_level & 2) { |
| 503 |
util().progress("\tEnd Surface Output"); |
| 504 |
} |
| 505 |
|
| 506 |
// All the rest of these depend on progressing through the cells in |
| 507 |
// correct order so that can pass information correctly from cell to |
| 508 |
// cell. Need to figure out how to eliminate this ordering so can |
| 509 |
// parallelize and not worry about open file count. |
| 510 |
|
| 511 |
for (size_t j = 0; j < JJ(); j++) { |
| 512 |
for (size_t i = 0; i < II(); i++) { |
| 513 |
auto &cell = get_cell(i, j); |
| 514 |
auto node_map = generate_node_map(*this, cell, Mode::PROCESSOR, INT(0)); |
| 515 |
output_block_connectivity(cell, node_map); |
| 516 |
if (parallel_size() > 1) { |
| 517 |
output_nodal_communication_map(cell, node_map); |
| 518 |
} |
| 519 |
} |
| 520 |
} |
| 521 |
if (debug_level & 2) { |
| 522 |
util().progress("\tEnd Nodal Communication Map Output"); |
| 523 |
} |
| 524 |
|
| 525 |
if (parallel_size() > 1) { |
| 526 |
for (size_t j = 0; j < JJ(); j++) { |
| 527 |
for (size_t i = 0; i < II(); i++) { |
| 528 |
auto &cell = get_cell(i, j); |
| 529 |
output_node_map(cell, INT(0)); |
| 530 |
output_element_map(cell, INT(0)); |
| 531 |
} |
| 532 |
} |
| 533 |
if (debug_level & 2) { |
| 534 |
util().progress("\tEnd Node/Element Map Output"); |
| 535 |
} |
| 536 |
} |
| 537 |
} |
| 538 |
|
| 539 |
void Grid::output_nodal_coordinates(const Cell &cell) |
| 540 |
{ |
| 541 |
int rank = cell.rank(Loc::C); |
| 542 |
|
| 543 |
auto *nb = cell.region()->get_node_blocks()[0]; |
| 544 |
std::vector<double> coord_x; |
| 545 |
std::vector<double> coord_y; |
| 546 |
std::vector<double> coord_z; |
| 547 |
|
| 548 |
// Are we modifying the coordinates ... scale and/or offset and/or offset_unit_cell... |
| 549 |
bool mod_x = cell.m_offX != 0.0 || m_scaleFactor != 1.0 || m_offset[0] != 0.0; |
| 550 |
bool mod_y = cell.m_offY != 0.0 || m_scaleFactor != 1.0 || m_offset[1] != 0.0; |
| 551 |
bool mod_z = m_scaleFactor != 1.0 || m_offset[2] != 0.0; |
| 552 |
|
| 553 |
double scale = m_scaleFactor; |
| 554 |
nb->get_field_data("mesh_model_coordinates_x", coord_x); |
| 555 |
if (mod_x) { |
| 556 |
double offset = m_offset[0]; |
| 557 |
std::for_each(coord_x.begin(), coord_x.end(), |
| 558 |
[&cell, scale, offset](double &d) { d = (d + cell.m_offX) * scale + offset; }); |
| 559 |
} |
| 560 |
|
| 561 |
nb->get_field_data("mesh_model_coordinates_y", coord_y); |
| 562 |
if (mod_y) { |
| 563 |
double offset = m_offset[1]; |
| 564 |
std::for_each(coord_y.begin(), coord_y.end(), |
| 565 |
[&cell, scale, offset](double &d) { d = (d + cell.m_offY) * scale + offset; }); |
| 566 |
} |
| 567 |
|
| 568 |
nb->get_field_data("mesh_model_coordinates_z", coord_z); |
| 569 |
if (mod_z) { |
| 570 |
double offset = m_offset[2]; |
| 571 |
std::for_each(coord_z.begin(), coord_z.end(), |
| 572 |
[scale, offset](double &d) { d = d * scale + offset; }); |
| 573 |
} |
| 574 |
|
| 575 |
// Filter coordinates down to only "new nodes"... |
| 576 |
if (m_equivalenceNodes && (cell.has_neighbor_i() || cell.has_neighbor_j())) { |
| 577 |
auto mode = parallel_size() > 1 ? Mode::PROCESSOR : Mode::GLOBAL; |
| 578 |
auto categorized_nodes = cell.categorize_nodes(mode); |
| 579 |
size_t nn = 0; |
| 580 |
for (size_t n = 0; n < categorized_nodes.size(); n++) { |
| 581 |
if (categorized_nodes[n] == 0) { |
| 582 |
coord_x[nn] = coord_x[n]; |
| 583 |
coord_y[nn] = coord_y[n]; |
| 584 |
coord_z[nn] = coord_z[n]; |
| 585 |
nn++; |
| 586 |
} |
| 587 |
} |
| 588 |
} |
| 589 |
|
| 590 |
int exoid = output_region(rank)->get_database()->get_file_pointer(); |
| 591 |
auto start = cell.m_localNodeIdOffset + 1; |
| 592 |
auto count = cell.added_node_count(Mode::PROCESSOR, m_equivalenceNodes); |
| 593 |
ex_put_partial_coord(exoid, start, count, Data(coord_x), Data(coord_y), Data(coord_z)); |
| 594 |
|
| 595 |
if (minimize_open_files(Minimize::UNIT)) { |
| 596 |
cell.region()->get_database()->closeDatabase(); |
| 597 |
} |
| 598 |
} |
| 599 |
|
| 600 |
template <typename INT> void Grid::output_generated_surfaces(Cell &cell, INT /*dummy*/) |
| 601 |
{ |
| 602 |
auto generated_ijk = get_generated_sidesets(); |
| 603 |
if (generated_ijk == 0) { |
| 604 |
return; |
| 605 |
} |
| 606 |
|
| 607 |
int rank = cell.rank(Loc::C); |
| 608 |
|
| 609 |
const std::array<int, 6> boundary_rank{ |
| 610 |
cell.rank(Loc::L), cell.rank(Loc::R), cell.rank(Loc::B), cell.rank(Loc::T), -1, -1}; |
| 611 |
std::array<enum Flg, 6> boundary_flag{Flg::MIN_I, Flg::MAX_I, Flg::MIN_J, |
| 612 |
Flg::MAX_J, Flg::MIN_K, Flg::MAX_K}; |
| 613 |
|
| 614 |
int exoid = output_region(rank)->get_database()->get_file_pointer(); |
| 615 |
|
| 616 |
for (int face = 0; face < 6; face++) { |
| 617 |
if ((generated_ijk & (unsigned)boundary_flag[face]) && boundary_rank[face] == -1) { |
| 618 |
// Find surface on output mesh... |
| 619 |
auto *osurf = output_region(rank)->get_sideset(generated_surface_names[face]); |
| 620 |
SMART_ASSERT(osurf != nullptr); |
| 621 |
const auto &oblocks = osurf->get_side_blocks(); |
| 622 |
SMART_ASSERT(oblocks.size() == 1)(oblocks.size()); |
| 623 |
|
| 624 |
auto &boundary = cell.unit()->boundary_blocks[face]; |
| 625 |
auto count = boundary.size(); |
| 626 |
std::vector<INT> elements; |
| 627 |
std::vector<INT> faces; |
| 628 |
elements.reserve(count); |
| 629 |
faces.reserve(count); |
| 630 |
|
| 631 |
for (const auto &block_faces : boundary.m_faces) { |
| 632 |
const auto &block_name = block_faces.first; |
| 633 |
const auto &bnd_faces = block_faces.second; |
| 634 |
|
| 635 |
// This is the offset within this element block -- i.e., the 'element_offsetth' element in |
| 636 |
// this block. |
| 637 |
size_t element_offset = cell.m_localElementIdOffset[block_name]; |
| 638 |
// This is the offset of the elements in this element block within all of the elements in |
| 639 |
// the output file. |
| 640 |
size_t global_offset = output_region(rank)->get_element_block(block_name)->get_offset(); |
| 641 |
|
| 642 |
for (const auto &bface : bnd_faces) { |
| 643 |
elements.push_back(bface / 10 + element_offset + global_offset); |
| 644 |
faces.push_back(bface % 10 + 1); |
| 645 |
} |
| 646 |
} |
| 647 |
|
| 648 |
auto id = osurf->get_property("id").get_int(); |
| 649 |
auto start = cell.m_localSurfaceOffset[generated_surface_names[face]]; |
| 650 |
ex_put_partial_set(exoid, EX_SIDE_SET, id, start + 1, count, Data(elements), Data(faces)); |
| 651 |
} |
| 652 |
} |
| 653 |
} |
| 654 |
|
| 655 |
template <typename INT> void Grid::output_surfaces(Cell &cell, INT /*dummy*/) |
| 656 |
{ |
| 657 |
int rank = cell.rank(Loc::C); |
| 658 |
int exoid = output_region(rank)->get_database()->get_file_pointer(); |
| 659 |
|
| 660 |
// Get the surfaces on this cell... |
| 661 |
const auto &surfaces = cell.region()->get_sidesets(); |
| 662 |
for (const auto *surface : surfaces) { |
| 663 |
|
| 664 |
// Find corresponding surface on output mesh... |
| 665 |
auto *osurf = output_region(rank)->get_sideset(surface->name()); |
| 666 |
SMART_ASSERT(osurf != nullptr); |
| 667 |
const auto &oblocks = osurf->get_side_blocks(); |
| 668 |
|
| 669 |
std::vector<INT> elements; |
| 670 |
std::vector<INT> faces; |
| 671 |
elements.reserve(oblocks[0]->entity_count()); |
| 672 |
faces.reserve(oblocks[0]->entity_count()); |
| 673 |
|
| 674 |
const auto &blocks = surface->get_side_blocks(); |
| 675 |
for (const auto *block : blocks) { |
| 676 |
// Get the element/face pairs for the SideBlock in this surface... |
| 677 |
std::vector<INT> element_side; |
| 678 |
block->get_field_data("element_side_raw", element_side); |
| 679 |
|
| 680 |
// Now separate the element and the face into different vectors and |
| 681 |
// update the element number to match the elements in this cell... |
| 682 |
const auto *parent = block->parent_element_block(); |
| 683 |
SMART_ASSERT(parent != nullptr); |
| 684 |
|
| 685 |
// This is the offset of the elements in the unit cell parent element block |
| 686 |
size_t unit_block_offset = parent->get_offset(); |
| 687 |
|
| 688 |
// This is the offset within this element block -- i.e., the 'element_offsetth' element in |
| 689 |
// this block. |
| 690 |
size_t element_offset = cell.m_localElementIdOffset[parent->name()]; |
| 691 |
// This is the offset of the elements in this element block within all of the elements in the |
| 692 |
// output file. |
| 693 |
size_t global_offset = output_region(rank)->get_element_block(parent->name())->get_offset(); |
| 694 |
|
| 695 |
size_t entity_count = block->entity_count(); |
| 696 |
for (size_t i = 0; i < entity_count; i++) { |
| 697 |
auto element = element_side[2 * i + 0]; |
| 698 |
auto output_element = element - unit_block_offset + element_offset + global_offset; |
| 699 |
elements.push_back(output_element); |
| 700 |
faces.push_back(element_side[2 * i + 1]); |
| 701 |
} |
| 702 |
} |
| 703 |
|
| 704 |
auto id = osurf->get_property("id").get_int(); |
| 705 |
auto start = cell.m_localSurfaceOffset[osurf->name()]; |
| 706 |
auto count = elements.size(); |
| 707 |
ex_put_partial_set(exoid, EX_SIDE_SET, id, start + 1, count, Data(elements), Data(faces)); |
| 708 |
} |
| 709 |
|
| 710 |
if (minimize_open_files(Minimize::UNIT)) { |
| 711 |
cell.region()->get_database()->closeDatabase(); |
| 712 |
} |
| 713 |
} |
| 714 |
|
| 715 |
template <typename INT> |
| 716 |
void Grid::output_block_connectivity(Cell &cell, const std::vector<INT> &node_map) |
| 717 |
{ |
| 718 |
int rank = cell.rank(Loc::C); |
| 719 |
if (rank >= m_startRank && rank < m_startRank + m_rankCount) { |
| 720 |
int exoid = output_region(rank)->get_database()->get_file_pointer(); |
| 721 |
|
| 722 |
const auto &blocks = cell.region()->get_element_blocks(); |
| 723 |
std::vector<INT> connect; |
| 724 |
for (const auto *block : blocks) { |
| 725 |
block->get_field_data("connectivity_raw", connect); |
| 726 |
for (size_t k = 0; k < connect.size(); k++) { |
| 727 |
connect[k] = node_map[connect[k]]; |
| 728 |
} |
| 729 |
auto start = cell.m_localElementIdOffset[block->name()] + 1; |
| 730 |
auto count = block->entity_count(); |
| 731 |
auto id = block->get_property("id").get_int(); |
| 732 |
if (debug_level & 8) { |
| 733 |
fmt::print(stderr, "Rank: {}, Cell({}, {}), Block {}, id {}, start {}, count {}\n", rank, |
| 734 |
cell.m_i, cell.m_j, block->name(), id, start, count); |
| 735 |
} |
| 736 |
ex_put_partial_conn(exoid, EX_ELEM_BLOCK, id, start, count, Data(connect), nullptr, nullptr); |
| 737 |
} |
| 738 |
|
| 739 |
if (debug_level & 2) { |
| 740 |
util().progress(fmt::format("Generated Node Map / Output Connectivity for Cell({}, {})", |
| 741 |
cell.m_i, cell.m_j)); |
| 742 |
} |
| 743 |
if (minimize_open_files(Minimize::UNIT)) { |
| 744 |
cell.region()->get_database()->closeDatabase(); |
| 745 |
} |
| 746 |
if (minimize_open_files(Minimize::OUTPUT)) { |
| 747 |
output_region(rank)->get_database()->closeDatabase(); |
| 748 |
} |
| 749 |
} |
| 750 |
} |
| 751 |
|
| 752 |
template <typename INT> |
| 753 |
void Grid::output_nodal_communication_map(Cell &cell, const std::vector<INT> &node_map) |
| 754 |
{ |
| 755 |
// The `node_map` has the processor-local node ids for all nodes on this cell. |
| 756 |
// Note that the `node_map` starts at index 1 in the vector... |
| 757 |
// Need to check the boundaries of the cell and determine which boundaries are on a different rank |
| 758 |
// and output the nodes and processors to the communication map. |
| 759 |
int rank = cell.rank(Loc::C); |
| 760 |
if (rank >= m_startRank && rank < m_startRank + m_rankCount) { |
| 761 |
std::vector<INT> nodes; |
| 762 |
std::vector<INT> procs; |
| 763 |
cell.populate_node_communication_map(node_map, nodes, procs); |
| 764 |
|
| 765 |
int exoid = output_region(rank)->get_database()->get_file_pointer(); |
| 766 |
|
| 767 |
auto start = cell.m_communicationNodeOffset + 1; |
| 768 |
auto count = cell.m_communicationNodeCount; |
| 769 |
|
| 770 |
ex_put_partial_node_cmap(exoid, 1, start, count, Data(nodes), Data(procs), rank); |
| 771 |
|
| 772 |
if (minimize_open_files(Minimize::OUTPUT)) { |
| 773 |
output_region(rank)->get_database()->closeDatabase(); |
| 774 |
} |
| 775 |
|
| 776 |
if (debug_level & 32) { |
| 777 |
fmt::print(stderr, "Rank: {}, Cell({}, {}), Node Comm Map: start {}, count {}\n", rank, |
| 778 |
cell.m_i, cell.m_j, start, count); |
| 779 |
} |
| 780 |
|
| 781 |
if (debug_level & 2) { |
| 782 |
util().progress( |
| 783 |
fmt::format("Output Nodal Communication Map for Cell({}, {})", cell.m_i, cell.m_j)); |
| 784 |
} |
| 785 |
} |
| 786 |
} |
| 787 |
|
| 788 |
template <typename INT> void Grid::output_node_map(const Cell &cell, INT /*dummy*/) |
| 789 |
{ |
| 790 |
int rank = cell.rank(Loc::C); |
| 791 |
|
| 792 |
auto start = cell.m_localNodeIdOffset + 1; |
| 793 |
auto count = cell.added_node_count(Mode::PROCESSOR, m_equivalenceNodes); |
| 794 |
|
| 795 |
if (parallel_size() == 1) { |
| 796 |
INT gid = cell.m_globalNodeIdOffset + 1; |
| 797 |
std::vector<INT> map(count); |
| 798 |
std::iota(map.begin(), map.end(), gid); |
| 799 |
int exoid = output_region(rank)->get_database()->get_file_pointer(); |
| 800 |
ex_put_partial_id_map(exoid, EX_NODE_MAP, start, count, Data(map)); |
| 801 |
} |
| 802 |
else { |
| 803 |
auto map = generate_node_map(*this, cell, Mode::GLOBAL, INT(0)); |
| 804 |
|
| 805 |
if (rank >= m_startRank && rank < m_startRank + m_rankCount) { |
| 806 |
|
| 807 |
// Filter nodes down to only "new nodes"... |
| 808 |
if (m_equivalenceNodes && (cell.has_neighbor_i() || cell.has_neighbor_j())) { |
| 809 |
auto mode = Mode::PROCESSOR; |
| 810 |
auto categorized_nodes = cell.categorize_nodes(mode); |
| 811 |
size_t nn = 0; |
| 812 |
for (size_t n = 0; n < categorized_nodes.size(); n++) { |
| 813 |
if (categorized_nodes[n] == 0) { |
| 814 |
map[nn + 1] = map[n + 1]; |
| 815 |
nn++; |
| 816 |
} |
| 817 |
} |
| 818 |
} |
| 819 |
if (debug_level & 8) { |
| 820 |
fmt::print("Cell({}, {}), start {}, count {}\n", cell.m_i, cell.m_j, start, count); |
| 821 |
} |
| 822 |
int exoid = output_region(rank)->get_database()->get_file_pointer(); |
| 823 |
ex_put_partial_id_map(exoid, EX_NODE_MAP, start, count, &map[1]); |
| 824 |
if (minimize_open_files(Minimize::OUTPUT)) { |
| 825 |
output_region(rank)->get_database()->closeDatabase(); |
| 826 |
} |
| 827 |
} |
| 828 |
} |
| 829 |
|
| 830 |
if (debug_level & 2) { |
| 831 |
util().progress(fmt::format("Generated Node Map for Rank {}, Cell({}, {}): start {}, count " |
| 832 |
"{}\n", |
| 833 |
rank, cell.m_i, cell.m_j, start, count)); |
| 834 |
} |
| 835 |
} |
| 836 |
|
| 837 |
template <typename INT> void Grid::output_element_map(Cell &cell, INT /*dummy*/) |
| 838 |
{ |
| 839 |
int rank = cell.rank(Loc::C); |
| 840 |
if (rank >= m_startRank && rank < m_startRank + m_rankCount) { |
| 841 |
int exoid = output_region(rank)->get_database()->get_file_pointer(); |
| 842 |
|
| 843 |
const auto &output_blocks = output_region(rank)->get_element_blocks(); |
| 844 |
|
| 845 |
// This is the element block offset for the "single output file" |
| 846 |
// for the block being output For example, if the total mesh has |
| 847 |
// 3 blocks, with 100, 200, 100 elements, then the element block |
| 848 |
// offset would be 0, 100, 300 (Ioss::ElementBlock::get_offset()) |
| 849 |
size_t global_id_offset = 0; |
| 850 |
|
| 851 |
for (const auto *output_element_block : output_blocks) { |
| 852 |
auto *block = cell.region()->get_element_block(output_element_block->name()); |
| 853 |
if (block != nullptr) { |
| 854 |
|
| 855 |
INT gid = cell.m_globalElementIdOffset[block->name()] + 1 + global_id_offset; |
| 856 |
auto element_count = block->entity_count(); |
| 857 |
std::vector<INT> map(element_count); |
| 858 |
|
| 859 |
std::iota(map.begin(), map.end(), gid); |
| 860 |
|
| 861 |
auto output_block_offset = output_element_block->get_offset(); |
| 862 |
|
| 863 |
// This cells element block ids start this far into the portion of the map for this |
| 864 |
// element block |
| 865 |
auto local_offset = cell.m_localElementIdOffset[block->name()]; |
| 866 |
|
| 867 |
auto start = output_block_offset + local_offset + 1; |
| 868 |
ex_put_partial_id_map(exoid, EX_ELEM_MAP, start, element_count, Data(map)); |
| 869 |
|
| 870 |
if (debug_level & 8) { |
| 871 |
fmt::print("Rank {}: Cell({}, {}), Block {}, start {}, element_count {}, gid {}\n", rank, |
| 872 |
cell.m_i, cell.m_j, block->name(), start, element_count, gid); |
| 873 |
} |
| 874 |
} |
| 875 |
// If we were outputting a single file, then this element |
| 876 |
// block in that file would have this many elements. |
| 877 |
auto global_block_element_count = |
| 878 |
output_element_block->get_property("global_entity_count").get_int(); |
| 879 |
global_id_offset += global_block_element_count; |
| 880 |
} |
| 881 |
if (minimize_open_files(Minimize::OUTPUT)) { |
| 882 |
output_region(rank)->get_database()->closeDatabase(); |
| 883 |
} |
| 884 |
} |
| 885 |
} |
| 886 |
|
| 887 |
void Grid::handle_file_count() |
| 888 |
{ |
| 889 |
// If the user has specified a 'minimize_open_files' behavior on the command line, |
| 890 |
// honor that mode here. Note that we have already processed the unit cells and |
| 891 |
// if the unit cell count exceeds the open file limit, then that code has already |
| 892 |
// automatically set the minimize_open_files to include UNIT... |
| 893 |
if (m_minimizeOpenFiles == Minimize::ALL) { |
| 894 |
return; |
| 895 |
} |
| 896 |
|
| 897 |
size_t open_files = open_file_limit(); |
| 898 |
if (util().parallel_rank() == 0) { |
| 899 |
fmt::print("\n Maximum Open File Count = {}\n", open_file_limit()); |
| 900 |
} |
| 901 |
|
| 902 |
auto unit_cell_size = unit_cells().size(); |
| 903 |
if (minimize_open_files(Minimize::UNIT)) { |
| 904 |
unit_cell_size = 1; // There will only be a single unit cell file open at a time. |
| 905 |
} |
| 906 |
|
| 907 |
if (unit_cell_size + m_rankCount > open_files) { |
| 908 |
// Too many files, need to close some of them after accessing... |
| 909 |
// We have already checked the unit_cell() size earlier. |
| 910 |
// If user has specified 'minimize_open_files == OUTPUT', then honor that... |
| 911 |
if (minimize_open_files(Minimize::OUTPUT)) { |
| 912 |
return; |
| 913 |
} |
| 914 |
|
| 915 |
// Have decision on whether to keep all unit_files open |
| 916 |
// and reduce open output files further, or to close |
| 917 |
// unit files and have more output files open... |
| 918 |
int output_open = open_files - (int)unit_cell_size; |
| 919 |
if (output_open < (int)(.2 * m_rankCount)) { |
| 920 |
// Close the unit files. Prefer to have output files open instead. |
| 921 |
m_minimizeOpenFiles = Minimize((unsigned)m_minimizeOpenFiles | (unsigned)Minimize::UNIT); |
| 922 |
unit_cell_size = 1; |
| 923 |
} |
| 924 |
|
| 925 |
// Set the rank count to the number of files left after subtracting off 'unit_cell_size' |
| 926 |
// and set the 'subcycle' mode. |
| 927 |
auto max_rank_count = open_files - unit_cell_size; |
| 928 |
if (max_rank_count < (size_t)m_rankCount) { |
| 929 |
m_rankCount = max_rank_count; |
| 930 |
} |
| 931 |
m_subCycle = true; |
| 932 |
} |
| 933 |
|
| 934 |
if (util().parallel_rank() == 0 && m_minimizeOpenFiles != Minimize::NONE) { |
| 935 |
std::array<std::string, 4> smode{"NONE", "UNIT", "OUTPUT", "ALL"}; |
| 936 |
fmt::print(" Setting `minimize_open_files` mode to {}.\n", smode[(int)m_minimizeOpenFiles]); |
| 937 |
} |
| 938 |
} |
| 939 |
|
| 940 |
namespace { |
| 941 |
size_t handle_elements(Grid &grid, int start_rank, int rank_count) |
| 942 |
{ |
| 943 |
// Not all unit cells have the same element blocks and the output |
| 944 |
// grid will contain the union of the element blocks on each unit |
| 945 |
// cell... |
| 946 |
// |
| 947 |
// While finalizing the cells, we will create this union for use in |
| 948 |
// the output region. Would be quicker to do iteration on unit_cell |
| 949 |
// map which is smaller, but for now do it here and see if becomes |
| 950 |
// bottleneck. |
| 951 |
|
| 952 |
std::map<std::string, std::unique_ptr<Ioss::ElementBlock>> output_element_blocks; |
| 953 |
std::vector<std::map<std::string, size_t>> element_block_elem_count(grid.parallel_size()); |
| 954 |
std::map<std::string, size_t> global_element_block_elem_count; |
| 955 |
|
| 956 |
for (size_t j = 0; j < grid.JJ(); j++) { |
| 957 |
for (size_t i = 0; i < grid.II(); i++) { |
| 958 |
auto &cell = grid.get_cell(i, j); |
| 959 |
auto rank = cell.rank(Loc::C); |
| 960 |
|
| 961 |
const auto &element_blocks = cell.region()->get_element_blocks(); |
| 962 |
for (const auto *block : element_blocks) { |
| 963 |
const auto &blk = block->name(); |
| 964 |
cell.m_globalElementIdOffset[blk] = global_element_block_elem_count[blk]; |
| 965 |
cell.m_localElementIdOffset[blk] = element_block_elem_count[rank][blk]; |
| 966 |
|
| 967 |
element_block_elem_count[rank][blk] += block->entity_count(); |
| 968 |
global_element_block_elem_count[blk] += block->entity_count(); |
| 969 |
|
| 970 |
if (debug_level & 8) { |
| 971 |
fmt::print("rank, i, j, blk, loffset, goffset, block_count: {}: {} {} {} {} {} {}\n", |
| 972 |
rank, i, j, blk, cell.m_localElementIdOffset[blk], |
| 973 |
cell.m_globalElementIdOffset[blk], element_block_elem_count[rank][blk]); |
| 974 |
} |
| 975 |
|
| 976 |
// Create output element block if does not exist yet... |
| 977 |
if (output_element_blocks.find(blk) == output_element_blocks.end()) { |
| 978 |
output_element_blocks.emplace( |
| 979 |
blk, std::make_unique<Ioss::ElementBlock>(Ioss::ElementBlock(*block))); |
| 980 |
} |
| 981 |
} |
| 982 |
} |
| 983 |
} |
| 984 |
|
| 985 |
// Calculate values needed to set the "global_entity_count" property on the output element |
| 986 |
// blocks. |
| 987 |
size_t global_element_count = 0; |
| 988 |
std::map<const std::string, int64_t> global_block_element_count; |
| 989 |
for (auto &blk : output_element_blocks) { |
| 990 |
for (int rank = 0; rank < grid.parallel_size(); rank++) { |
| 991 |
auto *block = blk.second.get(); |
| 992 |
global_block_element_count[block->name()] += element_block_elem_count[rank][block->name()]; |
| 993 |
global_element_count += element_block_elem_count[rank][block->name()]; |
| 994 |
} |
| 995 |
} |
| 996 |
|
| 997 |
// Define the element blocks in the output database... |
| 998 |
for (int rank = start_rank; rank < start_rank + rank_count; rank++) { |
| 999 |
for (auto &blk : output_element_blocks) { |
| 1000 |
auto *block = new Ioss::ElementBlock(*blk.second); |
| 1001 |
block->property_update("entity_count", element_block_elem_count[rank][block->name()]); |
| 1002 |
block->property_update("global_entity_count", global_block_element_count[block->name()]); |
| 1003 |
grid.output_region(rank)->property_add( |
| 1004 |
Ioss::Property("global_element_count", (int64_t)global_element_count)); |
| 1005 |
grid.output_region(rank)->add(block); |
| 1006 |
if (debug_level & 8) { |
| 1007 |
fmt::print("rank, blk, element_count: {}: {} {}\n", rank, block->name(), |
| 1008 |
block->entity_count()); |
| 1009 |
} |
| 1010 |
} |
| 1011 |
} |
| 1012 |
return global_element_count; |
| 1013 |
} |
| 1014 |
|
| 1015 |
size_t handle_nodes(Grid &grid, int start_rank, int rank_count) |
| 1016 |
{ |
| 1017 |
size_t global_node_count = 0; |
| 1018 |
std::vector<size_t> local_node_count(grid.parallel_size()); |
| 1019 |
|
| 1020 |
for (size_t j = 0; j < grid.JJ(); j++) { |
| 1021 |
for (size_t i = 0; i < grid.II(); i++) { |
| 1022 |
auto &cell = grid.get_cell(i, j); |
| 1023 |
auto rank = cell.rank(Loc::C); |
| 1024 |
cell.m_globalNodeIdOffset = global_node_count; |
| 1025 |
cell.m_localNodeIdOffset = local_node_count[rank]; |
| 1026 |
SMART_ASSERT(cell.region() != nullptr)(i)(j); |
| 1027 |
|
| 1028 |
auto new_global_nodes = cell.added_node_count(Mode::GLOBAL, grid.equivalence_nodes()); |
| 1029 |
auto new_processor_nodes = cell.added_node_count(Mode::PROCESSOR, grid.equivalence_nodes()); |
| 1030 |
if (debug_level & 8) { |
| 1031 |
fmt::print("rank: i, j, node_offset, added_nodes: {}: {} {} {} {} {}\n", rank, i, j, |
| 1032 |
local_node_count[rank], new_global_nodes, new_processor_nodes); |
| 1033 |
} |
| 1034 |
local_node_count[rank] += new_processor_nodes; |
| 1035 |
global_node_count += new_global_nodes; |
| 1036 |
} |
| 1037 |
} |
| 1038 |
// Define the output database node block... |
| 1039 |
for (int i = start_rank; i < start_rank + rank_count; i++) { |
| 1040 |
std::string block_name = "nodeblock_1"; |
| 1041 |
int spatial_dimension = 3; |
| 1042 |
auto *block = new Ioss::NodeBlock(grid.output_region(i)->get_database(), block_name, |
| 1043 |
local_node_count[i], spatial_dimension); |
| 1044 |
block->property_add(Ioss::Property("id", 1)); |
| 1045 |
grid.output_region(i)->add(block); |
| 1046 |
grid.output_region(i)->property_add( |
| 1047 |
Ioss::Property("global_node_count", (int64_t)global_node_count)); |
| 1048 |
} |
| 1049 |
return global_node_count; |
| 1050 |
} |
| 1051 |
|
| 1052 |
void handle_communications(Grid &grid, int start_rank, int rank_count) |
| 1053 |
{ |
| 1054 |
// Need to determine the number of nodes on the processor |
| 1055 |
// boundaries for each output database and create a CommSet... |
| 1056 |
std::vector<size_t> bnode(grid.parallel_size()); |
| 1057 |
for (size_t j = 0; j < grid.JJ(); j++) { |
| 1058 |
for (size_t i = 0; i < grid.II(); i++) { |
| 1059 |
auto &cell = grid.get_cell(i, j); |
| 1060 |
auto rank = cell.rank(Loc::C); |
| 1061 |
cell.m_communicationNodeOffset = bnode[rank]; |
| 1062 |
auto bnd = cell.processor_boundary_node_count(); |
| 1063 |
bnode[rank] += bnd; |
| 1064 |
if (debug_level & 32) { |
| 1065 |
fmt::print("rank: {}, Cell({}, {}): Boundary Count = {}, Total = {}\n", rank, i, j, bnd, |
| 1066 |
bnode[rank]); |
| 1067 |
} |
| 1068 |
} |
| 1069 |
} |
| 1070 |
|
| 1071 |
// Now add communication sets to all output databases... |
| 1072 |
for (int rank = start_rank; rank < start_rank + rank_count; rank++) { |
| 1073 |
auto *cs = new Ioss::CommSet(grid.output_region(rank)->get_database(), "commset_node", "node", |
| 1074 |
bnode[rank]); |
| 1075 |
grid.output_region(rank)->add(cs); |
| 1076 |
} |
| 1077 |
} |
| 1078 |
|
| 1079 |
void handle_surfaces(Grid &grid, int start_rank, int rank_count) |
| 1080 |
{ |
| 1081 |
// Not all unit cells have the same surfaces and the output |
| 1082 |
// grid will contain the union of the surfaces on each unit |
| 1083 |
// cell that has a surface on the boundary of the output mesh. |
| 1084 |
// |
| 1085 |
// While finalizing the cells, we will create this union for use in |
| 1086 |
// the output region. Would be quicker to do iteration on unit_cell |
| 1087 |
// map which is smaller, but for now do it here and see if becomes |
| 1088 |
// bottleneck. |
| 1089 |
std::map<std::string, std::unique_ptr<Ioss::SideSet>> output_surfaces; |
| 1090 |
std::map<std::string, size_t> global_surface_face_count; |
| 1091 |
|
| 1092 |
// Per rank... |
| 1093 |
std::vector<std::map<std::string, size_t>> surface_face_count(grid.parallel_size()); |
| 1094 |
std::vector<std::map<std::string, size_t>> local_surface_offset(grid.parallel_size()); |
| 1095 |
|
| 1096 |
for (size_t j = 0; j < grid.JJ(); j++) { |
| 1097 |
for (size_t i = 0; i < grid.II(); i++) { |
| 1098 |
auto &cell = grid.get_cell(i, j); |
| 1099 |
auto rank = cell.rank(Loc::C); |
| 1100 |
|
| 1101 |
const auto &surfaces = cell.region()->get_sidesets(); |
| 1102 |
for (const auto *surface : surfaces) { |
| 1103 |
const auto &surf = surface->name(); |
| 1104 |
cell.m_localSurfaceOffset[surf] = local_surface_offset[rank][surf]; |
| 1105 |
|
| 1106 |
const auto &blocks = surface->get_side_blocks(); |
| 1107 |
for (const auto *blk : blocks) { |
| 1108 |
surface_face_count[rank][surf] += blk->entity_count(); |
| 1109 |
global_surface_face_count[surf] += blk->entity_count(); |
| 1110 |
local_surface_offset[rank][surf] += blk->entity_count(); |
| 1111 |
} |
| 1112 |
|
| 1113 |
// Create output surface if does not exist yet... |
| 1114 |
if (output_surfaces.find(surf) == output_surfaces.end()) { |
| 1115 |
output_surfaces.emplace(surf, std::make_unique<Ioss::SideSet>(Ioss::SideSet(*surface))); |
| 1116 |
} |
| 1117 |
} |
| 1118 |
} |
| 1119 |
} |
| 1120 |
|
| 1121 |
// Define the surfaces in the output database... |
| 1122 |
for (int rank = start_rank; rank < start_rank + rank_count; rank++) { |
| 1123 |
for (auto &surf : output_surfaces) { |
| 1124 |
auto *surface = |
| 1125 |
new Ioss::SideSet(grid.output_region(rank)->get_database(), surf.second->name()); |
| 1126 |
auto *block = |
| 1127 |
new Ioss::SideBlock(grid.output_region(rank)->get_database(), surf.second->name(), |
| 1128 |
"quad4", "hex8", surface_face_count[rank][surface->name()]); |
| 1129 |
surface->add(block); |
| 1130 |
block->property_update("global_entity_count", global_surface_face_count[surface->name()]); |
| 1131 |
surface->property_update("global_entity_count", global_surface_face_count[surface->name()]); |
| 1132 |
block->property_update("distribution_factor_count", 0); |
| 1133 |
grid.output_region(rank)->add(surface); |
| 1134 |
} |
| 1135 |
} |
| 1136 |
} |
| 1137 |
|
| 1138 |
void generate_surfaces(Grid &grid, int start_rank, int rank_count) |
| 1139 |
{ |
| 1140 |
// The user may have requested that sidesets on one or more |
| 1141 |
// of the models boundary surfaces (min i,j,k and/or max i,j,k) |
| 1142 |
// be automatically generated (they don't exist on the input |
| 1143 |
// unit cells). Handle those sidesets here. |
| 1144 |
|
| 1145 |
auto generated_ijk = grid.get_generated_sidesets(); |
| 1146 |
if (generated_ijk == 0) { |
| 1147 |
return; |
| 1148 |
} |
| 1149 |
|
| 1150 |
std::array<size_t, 6> global_surface_face_count{}; |
| 1151 |
|
| 1152 |
// Per rank... |
| 1153 |
std::vector<std::array<size_t, 6>> surface_face_count(grid.parallel_size()); |
| 1154 |
std::vector<std::array<size_t, 6>> local_surface_offset(grid.parallel_size()); |
| 1155 |
|
| 1156 |
std::array<enum Flg, 6> boundary_flag{Flg::MIN_I, Flg::MAX_I, Flg::MIN_J, |
| 1157 |
Flg::MAX_J, Flg::MIN_K, Flg::MAX_K}; |
| 1158 |
|
| 1159 |
for (size_t j = 0; j < grid.JJ(); j++) { |
| 1160 |
for (size_t i = 0; i < grid.II(); i++) { |
| 1161 |
auto &cell = grid.get_cell(i, j); |
| 1162 |
auto rank = cell.rank(Loc::C); |
| 1163 |
const std::array<int, 6> boundary_rank{ |
| 1164 |
cell.rank(Loc::L), cell.rank(Loc::R), cell.rank(Loc::B), cell.rank(Loc::T), -1, -1}; |
| 1165 |
|
| 1166 |
for (int face = 0; face < 6; face++) { |
| 1167 |
if ((generated_ijk & (unsigned)boundary_flag[face]) && boundary_rank[face] == -1) { |
| 1168 |
cell.m_localSurfaceOffset[grid.generated_surface_names[face]] = |
| 1169 |
local_surface_offset[rank][face]; |
| 1170 |
|
| 1171 |
auto count = cell.unit()->boundary_blocks[face].size(); |
| 1172 |
global_surface_face_count[face] += count; |
| 1173 |
surface_face_count[rank][face] += count; |
| 1174 |
local_surface_offset[rank][face] += count; |
| 1175 |
} |
| 1176 |
} |
| 1177 |
} |
| 1178 |
} |
| 1179 |
|
| 1180 |
// Define the surfaces in the output database... |
| 1181 |
for (int rank = start_rank; rank < start_rank + rank_count; rank++) { |
| 1182 |
for (size_t i = 0; i < 6; i++) { |
| 1183 |
if (global_surface_face_count[i] > 0) { |
| 1184 |
auto *surface = new Ioss::SideSet(grid.output_region(rank)->get_database(), |
| 1185 |
grid.generated_surface_names[i]); |
| 1186 |
auto *block = new Ioss::SideBlock(grid.output_region(rank)->get_database(), |
| 1187 |
grid.generated_surface_names[i], "quad4", "hex8", |
| 1188 |
surface_face_count[rank][i]); |
| 1189 |
surface->property_update("global_entity_count", global_surface_face_count[i]); |
| 1190 |
block->property_update("global_entity_count", global_surface_face_count[i]); |
| 1191 |
block->property_update("distribution_factor_count", 0); |
| 1192 |
surface->add(block); |
| 1193 |
grid.output_region(rank)->add(surface); |
| 1194 |
} |
| 1195 |
} |
| 1196 |
} |
| 1197 |
} |
| 1198 |
|
| 1199 |
template <typename INT> |
| 1200 |
std::vector<INT> generate_node_map(Grid &grid, const Cell &cell, Mode mode, INT /*dummy*/) |
| 1201 |
{ |
| 1202 |
// Generate a "map" from nodes in the input connectivity to the |
| 1203 |
// output connectivity in global nodes. If no neighbors, then |
| 1204 |
// this would just be adding `cell.m_globalNodeIdOffset` to each |
| 1205 |
// connectivity entry. |
| 1206 |
|
| 1207 |
std::vector<INT> map = cell.generate_node_map(mode, grid.equivalence_nodes(), INT(0)); |
| 1208 |
if (debug_level & 8) { |
| 1209 |
fmt::print("Cell({},{}) PROCESSOR MAP: {}\n", cell.m_i, cell.m_j, fmt::join(map, " ")); |
| 1210 |
} |
| 1211 |
|
| 1212 |
// Now that we have the node map for this cell, we need to save |
| 1213 |
// the mappings for the max_I and max_J faces and max_I-max_J edge |
| 1214 |
// for use by later neighbors... Check whether cell has neighbors |
| 1215 |
// on max_I or max_J faces... |
| 1216 |
if (grid.equivalence_nodes()) { |
| 1217 |
if ((mode == Mode::GLOBAL || |
| 1218 |
(mode == Mode::PROCESSOR && cell.rank(Loc::C) == cell.rank(Loc::R))) && |
| 1219 |
(cell.m_i + 1 < grid.II())) { |
| 1220 |
const auto &neighbor = grid.get_cell(cell.m_i + 1, cell.m_j); |
| 1221 |
cell.populate_neighbor(Loc::L, map, neighbor); |
| 1222 |
} |
| 1223 |
|
| 1224 |
if ((mode == Mode::GLOBAL || |
| 1225 |
(mode == Mode::PROCESSOR && cell.rank(Loc::C) == cell.rank(Loc::T))) && |
| 1226 |
(cell.m_j + 1 < grid.JJ())) { |
| 1227 |
const auto &neighbor = grid.get_cell(cell.m_i, cell.m_j + 1); |
| 1228 |
cell.populate_neighbor(Loc::B, map, neighbor); |
| 1229 |
} |
| 1230 |
|
| 1231 |
if (mode == Mode::PROCESSOR) { |
| 1232 |
if (cell.processor_boundary(Loc::L) && (cell.rank(Loc::TL) == cell.rank(Loc::C))) { |
| 1233 |
const auto &tl_corner = grid.get_cell(cell.m_i - 1, cell.m_j + 1); |
| 1234 |
cell.populate_neighbor(Loc::BR, map, tl_corner); |
| 1235 |
} |
| 1236 |
// Now the other "corner case" |
| 1237 |
if (cell.processor_boundary(Loc::R) && (cell.rank(Loc::TR) == cell.rank(Loc::C))) { |
| 1238 |
const auto &tr_corner = grid.get_cell(cell.m_i + 1, cell.m_j + 1); |
| 1239 |
cell.populate_neighbor(Loc::BL, map, tr_corner); |
| 1240 |
} |
| 1241 |
} |
| 1242 |
} |
| 1243 |
return map; |
| 1244 |
} |
| 1245 |
|
| 1246 |
Ioss::PropertyManager parse_properties(SystemInterface &interFace, int int_size) |
| 1247 |
{ |
| 1248 |
Ioss::PropertyManager properties; |
| 1249 |
if (int_size == 8) { |
| 1250 |
properties.add(Ioss::Property("INTEGER_SIZE_DB", 8)); |
| 1251 |
properties.add(Ioss::Property("INTEGER_SIZE_API", 8)); |
| 1252 |
} |
| 1253 |
|
| 1254 |
if (interFace.use_netcdf4()) { |
| 1255 |
properties.add(Ioss::Property("FILE_TYPE", "netcdf4")); |
| 1256 |
} |
| 1257 |
|
| 1258 |
if (interFace.use_netcdf5()) { |
| 1259 |
properties.add(Ioss::Property("FILE_TYPE", "netcdf5")); |
| 1260 |
} |
| 1261 |
|
| 1262 |
if (interFace.compression_level() > 0 || interFace.szip()) { |
| 1263 |
properties.add(Ioss::Property("FILE_TYPE", "netcdf4")); |
| 1264 |
properties.add(Ioss::Property("COMPRESSION_LEVEL", interFace.compression_level())); |
| 1265 |
properties.add(Ioss::Property("COMPRESSION_SHUFFLE", true)); |
| 1266 |
if (interFace.szip()) { |
| 1267 |
properties.add(Ioss::Property("COMPRESSION_METHOD", "szip")); |
| 1268 |
} |
| 1269 |
else if (interFace.zlib()) { |
| 1270 |
properties.add(Ioss::Property("COMPRESSION_METHOD", "zlib")); |
| 1271 |
} |
| 1272 |
} |
| 1273 |
return properties; |
| 1274 |
} |
| 1275 |
|
| 1276 |
std::shared_ptr<Ioss::Region> create_input_region(const std::string &key, std::string filename, |
| 1277 |
bool ints_32_bits) |
| 1278 |
{ |
| 1279 |
// Check that 'filename' does not contain a starting/ending double quote... |
| 1280 |
filename.erase(remove(filename.begin(), filename.end(), '\"'), filename.end()); |
| 1281 |
Ioss::DatabaseIO *dbi = Ioss::IOFactory::create("exodus", filename, Ioss::READ_RESTART, |
| 1282 |
Ioss::ParallelUtils::comm_self()); |
| 1283 |
if (dbi == nullptr || !dbi->ok(true)) { |
| 1284 |
std::exit(EXIT_FAILURE); |
| 1285 |
} |
| 1286 |
|
| 1287 |
dbi->set_surface_split_type(Ioss::SPLIT_BY_DONT_SPLIT); |
| 1288 |
if (ints_32_bits) { |
| 1289 |
dbi->set_int_byte_size_api(Ioss::USE_INT32_API); |
| 1290 |
} |
| 1291 |
else { |
| 1292 |
dbi->set_int_byte_size_api(Ioss::USE_INT64_API); |
| 1293 |
} |
| 1294 |
|
| 1295 |
// Splitting surfaces by element block makes it easier to transform the input |
| 1296 |
// element id into the output element ids. |
| 1297 |
dbi->set_surface_split_type(Ioss::SPLIT_BY_ELEMENT_BLOCK); |
| 1298 |
|
| 1299 |
// Generate a name for the region based on the key... |
| 1300 |
std::string name = "Region_" + key; |
| 1301 |
// NOTE: region owns database pointer at this time... |
| 1302 |
return std::make_shared<Ioss::Region>(dbi, name); |
| 1303 |
} |
| 1304 |
|
| 1305 |
void output_summary(Ioss::Region *region, std::ostream &strm) |
| 1306 |
{ |
| 1307 |
int64_t nodes = region->get_property("node_count").get_int(); |
| 1308 |
int64_t elements = region->get_property("element_count").get_int(); |
| 1309 |
|
| 1310 |
fmt::print(strm, " Database: {}\tNodes = {} \tElements = {}\n", |
| 1311 |
region->get_database()->get_filename(), fmt::group_digits(nodes), |
| 1312 |
fmt::group_digits(elements)); |
| 1313 |
} |
| 1314 |
|
| 1315 |
} // namespace |