| 1 |
// Copyright(C) 1999-2024 National Technology & Engineering Solutions |
| 2 |
// of Sandia, LLC (NTESS). Under the terms of Contract DE-NA0003525 with |
| 3 |
// NTESS, the U.S. Government retains certain rights in this software. |
| 4 |
// |
| 5 |
// See packages/seacas/LICENSE for details |
| 6 |
|
| 7 |
#include <SL_Decompose.h> |
| 8 |
#include <SL_SystemInterface.h> |
| 9 |
#include <SL_Version.h> |
| 10 |
|
| 11 |
#include <Ioss_CodeTypes.h> |
| 12 |
#include <Ioss_CopyDatabase.h> |
| 13 |
#include <Ioss_DatabaseIO.h> |
| 14 |
#include <Ioss_DecompositionUtils.h> |
| 15 |
#include <Ioss_FileInfo.h> |
| 16 |
#include <Ioss_MemoryUtils.h> |
| 17 |
#include <Ioss_MeshCopyOptions.h> |
| 18 |
#include <Ioss_Region.h> |
| 19 |
#include <Ioss_SubSystem.h> |
| 20 |
#include <Ioss_SurfaceSplit.h> |
| 21 |
#include <Ioss_Utils.h> |
| 22 |
#include <cassert> |
| 23 |
#include <fmt/color.h> |
| 24 |
#include <fmt/format.h> |
| 25 |
#include <fmt/ostream.h> |
| 26 |
#include <init/Ionit_Initializer.h> |
| 27 |
|
| 28 |
#include <exodusII.h> |
| 29 |
|
| 30 |
#include <algorithm> |
| 31 |
#include <array> |
| 32 |
#include <chrono> |
| 33 |
#include <cstddef> |
| 34 |
#include <cstdlib> |
| 35 |
#include <cstring> |
| 36 |
#include <map> |
| 37 |
#include <numeric> |
| 38 |
#include <string> |
| 39 |
#include <vector> |
| 40 |
|
| 41 |
#include <sys/types.h> |
| 42 |
|
| 43 |
#ifdef SEACAS_HAVE_MPI |
| 44 |
#include <mpi.h> |
| 45 |
#endif |
| 46 |
|
| 47 |
// ======================================================================== |
| 48 |
// TODO(gdsjaar): |
| 49 |
// * Sideset distribution factors |
| 50 |
// * Variables |
| 51 |
// * All entity types |
| 52 |
// * More efficient border-node-processor communication map. |
| 53 |
// ======================================================================== |
| 54 |
|
| 55 |
extern double seacas_timer(); |
| 56 |
int debug_level = 0; |
| 57 |
|
| 58 |
// size_t partial_count = 1'00'000; |
| 59 |
size_t partial_count = 1'000'000'000; |
| 60 |
|
| 61 |
void progress(const std::string &output) |
| 62 |
{ |
| 63 |
static auto start = std::chrono::steady_clock::now(); |
| 64 |
|
| 65 |
if ((debug_level & 1) != 0) { |
| 66 |
auto now = std::chrono::steady_clock::now(); |
| 67 |
std::chrono::duration<double> diff = now - start; |
| 68 |
fmt::print(stderr, " [{:.2f} - {}]\t{}\n", diff.count(), |
| 69 |
fmt::group_digits(Ioss::MemoryUtils::get_memory_info()), output); |
| 70 |
} |
| 71 |
} |
| 72 |
|
| 73 |
namespace { |
| 74 |
void proc_progress(int p, int proc_count) |
| 75 |
{ |
| 76 |
if (((debug_level & 8) != 0) && ((proc_count <= 20) || ((p + 1) % (proc_count / 20) == 0))) { |
| 77 |
progress("\t\tProcessor " + std::to_string(p + 1)); |
| 78 |
} |
| 79 |
} |
| 80 |
|
| 81 |
Ioss::PropertyManager set_properties(const SystemInterface &interFace) |
| 82 |
{ |
| 83 |
Ioss::PropertyManager properties; |
| 84 |
if (interFace.netcdf4_) { |
| 85 |
properties.add(Ioss::Property("FILE_TYPE", "netcdf4")); |
| 86 |
} |
| 87 |
|
| 88 |
if (interFace.netcdf5_) { |
| 89 |
properties.add(Ioss::Property("FILE_TYPE", "netcdf5")); |
| 90 |
} |
| 91 |
|
| 92 |
if (interFace.compressionLevel_ > 0 || interFace.shuffle_ || interFace.szip_ || |
| 93 |
interFace.zlib_ || interFace.zstd_) { |
| 94 |
properties.add(Ioss::Property("FILE_TYPE", "netcdf4")); |
| 95 |
properties.add(Ioss::Property("COMPRESSION_LEVEL", interFace.compressionLevel_)); |
| 96 |
properties.add(Ioss::Property("COMPRESSION_SHUFFLE", static_cast<int>(interFace.shuffle_))); |
| 97 |
|
| 98 |
if (interFace.zlib_) { |
| 99 |
properties.add(Ioss::Property("COMPRESSION_METHOD", "zlib")); |
| 100 |
} |
| 101 |
else if (interFace.szip_) { |
| 102 |
properties.add(Ioss::Property("COMPRESSION_METHOD", "szip")); |
| 103 |
} |
| 104 |
else if (interFace.zstd_) { |
| 105 |
properties.add(Ioss::Property("COMPRESSION_METHOD", "zstd")); |
| 106 |
} |
| 107 |
else if (interFace.bz2_) { |
| 108 |
properties.add(Ioss::Property("COMPRESSION_METHOD", "bzip2")); |
| 109 |
} |
| 110 |
|
| 111 |
if (interFace.quantizeNSD_ > 0) { |
| 112 |
properties.add(Ioss::Property("COMPRESSION_QUANTIZE_NSD", interFace.quantizeNSD_)); |
| 113 |
} |
| 114 |
} |
| 115 |
|
| 116 |
if (interFace.ints64Bit_) { |
| 117 |
properties.add(Ioss::Property("INTEGER_SIZE_DB", 8)); |
| 118 |
properties.add(Ioss::Property("INTEGER_SIZE_API", 8)); |
| 119 |
} |
| 120 |
return properties; |
| 121 |
} |
| 122 |
|
| 123 |
// Add the chain maps for file-per-rank output... |
| 124 |
template <typename INT> |
| 125 |
void output_chain_maps(std::vector<Ioss::Region *> &proc_region, const Ioss::chain_t<INT> &chains, |
| 126 |
const std::vector<int> &elem_to_proc, size_t proc_begin, size_t proc_size, |
| 127 |
INT /* dummy */) |
| 128 |
{ |
| 129 |
progress(__func__); |
| 130 |
|
| 131 |
// Build the proc-local chain map and then output it on a block-by-block basis... |
| 132 |
size_t proc_count = proc_region.size(); |
| 133 |
std::vector<std::vector<INT>> map(proc_count); |
| 134 |
|
| 135 |
size_t global_element_count = elem_to_proc.size(); |
| 136 |
for (size_t j = 0; j < global_element_count; j++) { |
| 137 |
size_t p = elem_to_proc[j]; |
| 138 |
if (p >= proc_begin && p < proc_begin + proc_size) { |
| 139 |
auto &chain_entry = chains[j]; |
| 140 |
// TODO: Map this from global to local element number... |
| 141 |
size_t loc_elem = |
| 142 |
chain_entry.element > 0 |
| 143 |
? proc_region[p]->get_database()->element_global_to_local(chain_entry.element) |
| 144 |
: 0; |
| 145 |
map[p].push_back(loc_elem); |
| 146 |
map[p].push_back(chain_entry.link); |
| 147 |
} |
| 148 |
} |
| 149 |
|
| 150 |
// Now iterate through the blocks on each rank and output that portion of the map... |
| 151 |
size_t block_count = proc_region[0]->get_property("element_block_count").get_int(); |
| 152 |
|
| 153 |
for (size_t p = proc_begin; p < proc_begin + proc_size; p++) { |
| 154 |
const auto &proc_ebs = proc_region[p]->get_element_blocks(); |
| 155 |
size_t offset = 0; |
| 156 |
for (size_t b = 0; b < block_count; b++) { |
| 157 |
size_t proc_element_count = proc_ebs[b]->entity_count(); |
| 158 |
proc_ebs[b]->put_field_data("chain", &map[p][offset], proc_element_count * 2 * sizeof(INT)); |
| 159 |
offset += proc_element_count; |
| 160 |
} |
| 161 |
} |
| 162 |
} |
| 163 |
|
| 164 |
void add_chain_maps(Ioss::Region ®ion) |
| 165 |
{ |
| 166 |
ex_put_map_param(region.get_database()->get_file_pointer(), 0, 2); |
| 167 |
ex_put_name(region.get_database()->get_file_pointer(), EX_ELEM_MAP, 1, "chain:root_element_id"); |
| 168 |
ex_put_name(region.get_database()->get_file_pointer(), EX_ELEM_MAP, 2, "chain:depth_from_root"); |
| 169 |
|
| 170 |
// The chain / line data will be stored as an element map... |
| 171 |
const auto &blocks = region.get_element_blocks(); |
| 172 |
for (const auto &block : blocks) { |
| 173 |
auto field = |
| 174 |
Ioss::Field("chain", region.field_int_type(), "Real[2]", Ioss::Field::MAP).set_index(1); |
| 175 |
block->field_add(std::move(field)); |
| 176 |
} |
| 177 |
} |
| 178 |
|
| 179 |
void add_decomp_map(Ioss::Region ®ion, const std::string &decomp_variable_name, |
| 180 |
bool add_chain_info) |
| 181 |
{ |
| 182 |
ex_opts(EX_VERBOSE); |
| 183 |
if (add_chain_info) { |
| 184 |
ex_put_map_param(region.get_database()->get_file_pointer(), 0, 3); |
| 185 |
ex_put_name(region.get_database()->get_file_pointer(), EX_ELEM_MAP, 1, |
| 186 |
decomp_variable_name.c_str()); |
| 187 |
ex_put_name(region.get_database()->get_file_pointer(), EX_ELEM_MAP, 2, |
| 188 |
"chain:root_element_id"); |
| 189 |
ex_put_name(region.get_database()->get_file_pointer(), EX_ELEM_MAP, 3, |
| 190 |
"chain:depth_from_root"); |
| 191 |
} |
| 192 |
else { |
| 193 |
ex_put_map_param(region.get_database()->get_file_pointer(), 0, 1); |
| 194 |
ex_put_name(region.get_database()->get_file_pointer(), EX_ELEM_MAP, 1, |
| 195 |
decomp_variable_name.c_str()); |
| 196 |
} |
| 197 |
|
| 198 |
// The chain / line data will be stored as an element map... |
| 199 |
const auto &blocks = region.get_element_blocks(); |
| 200 |
for (const auto &block : blocks) { |
| 201 |
auto field = |
| 202 |
Ioss::Field(decomp_variable_name, Ioss::Field::INT32, IOSS_SCALAR(), Ioss::Field::MAP) |
| 203 |
.set_index(1); |
| 204 |
block->field_add(std::move(field)); |
| 205 |
if (add_chain_info) { |
| 206 |
auto ch_field = |
| 207 |
Ioss::Field("chain", region.field_int_type(), "Real[2]", Ioss::Field::MAP).set_index(2); |
| 208 |
block->field_add(std::move(ch_field)); |
| 209 |
} |
| 210 |
} |
| 211 |
} |
| 212 |
|
| 213 |
template <typename INT> |
| 214 |
void output_decomp_map(Ioss::Region ®ion, const std::vector<int> &elem_to_proc, |
| 215 |
const Ioss::chain_t<INT> &chains, const std::string &decomp_variable_name, |
| 216 |
bool add_chain_info) |
| 217 |
{ |
| 218 |
const auto &blocks = region.get_element_blocks(); |
| 219 |
size_t offset = 0; |
| 220 |
|
| 221 |
// Convert is outputting to 64-bit database... |
| 222 |
std::vector<INT> elem_to_proc64; |
| 223 |
if (sizeof(INT) == 8) { |
| 224 |
elem_to_proc64.reserve(elem_to_proc.size()); |
| 225 |
for (const auto &etp : elem_to_proc) { |
| 226 |
elem_to_proc64.push_back(etp); |
| 227 |
} |
| 228 |
} |
| 229 |
for (const auto &block : blocks) { |
| 230 |
size_t num_elem = block->entity_count(); |
| 231 |
if (sizeof(INT) == 8) { |
| 232 |
block->put_field_data(decomp_variable_name, (void *)&elem_to_proc64[offset], -1); |
| 233 |
} |
| 234 |
else { |
| 235 |
block->put_field_data(decomp_variable_name, (void *)&elem_to_proc[offset], -1); |
| 236 |
} |
| 237 |
if (add_chain_info) { |
| 238 |
std::vector<INT> chain; |
| 239 |
chain.reserve(num_elem * 2); |
| 240 |
for (size_t i = 0; i < num_elem; i++) { |
| 241 |
auto &chain_entry = chains[i + offset]; |
| 242 |
chain.push_back(chain_entry.element); |
| 243 |
chain.push_back(chain_entry.link); |
| 244 |
} |
| 245 |
block->put_field_data("chain", chain); |
| 246 |
} |
| 247 |
offset += num_elem; |
| 248 |
} |
| 249 |
} |
| 250 |
|
| 251 |
void add_decomp_field(Ioss::Region ®ion, const std::string &decomp_variable_name, |
| 252 |
bool add_chain_info) |
| 253 |
{ |
| 254 |
region.begin_mode(Ioss::STATE_DEFINE_TRANSIENT); |
| 255 |
|
| 256 |
const auto &blocks = region.get_element_blocks(); |
| 257 |
for (const auto &block : blocks) { |
| 258 |
block->field_add(Ioss::Field(decomp_variable_name, region.field_int_type(), IOSS_SCALAR(), |
| 259 |
Ioss::Field::TRANSIENT)); |
| 260 |
if (add_chain_info) { |
| 261 |
block->field_add( |
| 262 |
Ioss::Field("chain", region.field_int_type(), "Real[2]", Ioss::Field::TRANSIENT)); |
| 263 |
} |
| 264 |
} |
| 265 |
region.end_mode(Ioss::STATE_DEFINE_TRANSIENT); |
| 266 |
} |
| 267 |
|
| 268 |
template <typename INT> |
| 269 |
void output_decomp_field(Ioss::Region ®ion, const std::vector<int> &elem_to_proc, |
| 270 |
const Ioss::chain_t<INT> &chains, |
| 271 |
const std::string &decomp_variable_name, bool add_chain_info) |
| 272 |
{ |
| 273 |
region.begin_mode(Ioss::STATE_TRANSIENT); |
| 274 |
|
| 275 |
auto step = region.add_state(0.0); |
| 276 |
region.begin_state(step); |
| 277 |
|
| 278 |
output_decomp_map(region, elem_to_proc, chains, decomp_variable_name, add_chain_info); |
| 279 |
|
| 280 |
region.end_state(step); |
| 281 |
region.end_mode(Ioss::STATE_TRANSIENT); |
| 282 |
} |
| 283 |
|
| 284 |
template <typename INT> |
| 285 |
void populate_proc_node(size_t count, size_t offset, size_t element_nodes, |
| 286 |
const std::vector<int> &elem_to_proc, const std::vector<INT> &glob_conn, |
| 287 |
std::vector<std::vector<int>> &proc_node, |
| 288 |
std::vector<size_t> &on_proc_count) |
| 289 |
{ |
| 290 |
// Determine which processor(s) each node is present on. |
| 291 |
// Also count number of nodes on each processor. |
| 292 |
size_t el = 0; |
| 293 |
for (size_t j = 0; j < count; j++) { |
| 294 |
auto p = elem_to_proc[offset + j]; |
| 295 |
for (size_t k = 0; k < element_nodes; k++) { |
| 296 |
INT node = glob_conn[el++] - 1; |
| 297 |
bool exists = std::find(std::begin(proc_node[node]), std::end(proc_node[node]), p) != |
| 298 |
std::end(proc_node[node]); |
| 299 |
if (!exists) { |
| 300 |
proc_node[node].push_back(p); |
| 301 |
on_proc_count[p]++; |
| 302 |
} |
| 303 |
} |
| 304 |
} |
| 305 |
} |
| 306 |
|
| 307 |
void filename_substitution(std::string &filename, const SystemInterface &interFace); |
| 308 |
|
| 309 |
template <typename INT> |
| 310 |
void slice(Ioss::Region ®ion, const std::string &nemfile, SystemInterface &interFace, |
| 311 |
INT dummy); |
| 312 |
|
| 313 |
template <typename INT> bool is_sequential(const std::vector<INT> &map) |
| 314 |
{ |
| 315 |
progress(__func__); |
| 316 |
for (size_t i = 0; i < map.size(); i++) { |
| 317 |
if (map[i] != i + 1) { |
| 318 |
return false; |
| 319 |
} |
| 320 |
} |
| 321 |
return true; |
| 322 |
} |
| 323 |
|
| 324 |
} // namespace |
| 325 |
// ======================================================================== |
| 326 |
|
| 327 |
int main(int argc, char *argv[]) |
| 328 |
{ |
| 329 |
#ifdef SEACAS_HAVE_MPI |
| 330 |
MPI_Init(&argc, &argv); |
| 331 |
#endif |
| 332 |
|
| 333 |
double begin = seacas_timer(); |
| 334 |
|
| 335 |
Ioss::Init::Initializer io; |
| 336 |
SystemInterface::show_version(); |
| 337 |
|
| 338 |
SystemInterface interFace; |
| 339 |
bool ok = interFace.parse_options(argc, argv); |
| 340 |
if (!ok) { |
| 341 |
fmt::print(stderr, "\nERROR: Problem parsing command line options.\n\n"); |
| 342 |
exit(EXIT_FAILURE); |
| 343 |
} |
| 344 |
|
| 345 |
std::string nem_file = interFace.nemesisFile_; |
| 346 |
std::string path = interFace.output_path(); |
| 347 |
if (!path.empty()) { |
| 348 |
filename_substitution(path, interFace); |
| 349 |
|
| 350 |
// See if specified path exists. |
| 351 |
Ioss::FileInfo output_path(path); |
| 352 |
if (!output_path.exists()) { |
| 353 |
// Try to create the directory... |
| 354 |
Ioss::FileInfo::create_path(path); |
| 355 |
} |
| 356 |
else if (!output_path.is_dir()) { |
| 357 |
fmt::print(stderr, "ERROR: Path '{}' is not a directory.\n", path); |
| 358 |
exit(EXIT_FAILURE); |
| 359 |
} |
| 360 |
|
| 361 |
// See if the nem_file already has a path prepended to the |
| 362 |
// filename and if so, extract the basename. |
| 363 |
Ioss::FileInfo nemesis(nem_file); |
| 364 |
std::string sep = "/"; |
| 365 |
if (path[path.length() - 1] == '/') { |
| 366 |
sep = ""; |
| 367 |
} |
| 368 |
nem_file = path + sep + nemesis.tailname(); |
| 369 |
} |
| 370 |
|
| 371 |
if (interFace.outputDecompMap_ || interFace.outputDecompField_) { |
| 372 |
// Then not creating split files, just adding map or field to a single output file |
| 373 |
// Need to check that not overwriting input file... |
| 374 |
if (interFace.inputFile_ == nem_file) { |
| 375 |
nem_file += "-decomp"; |
| 376 |
} |
| 377 |
} |
| 378 |
|
| 379 |
fmt::print(stderr, "\nInput: '{}'\n", interFace.inputFile_); |
| 380 |
fmt::print(stderr, "Output: '{}'\n", nem_file); |
| 381 |
|
| 382 |
debug_level = interFace.debug(); |
| 383 |
partial_count = interFace.partial(); |
| 384 |
|
| 385 |
//======================================================================== |
| 386 |
// INPUT ... |
| 387 |
// NOTE: The "READ_RESTART" mode ensures that the node and element ids will be mapped. |
| 388 |
//======================================================================== |
| 389 |
Ioss::DatabaseIO *dbi = |
| 390 |
Ioss::IOFactory::create(interFace.inputFormat_, interFace.inputFile_, Ioss::READ_RESTART, |
| 391 |
Ioss::ParallelUtils::comm_world()); |
| 392 |
if (dbi == nullptr || !dbi->ok(true)) { |
| 393 |
std::exit(EXIT_FAILURE); |
| 394 |
} |
| 395 |
|
| 396 |
if (interFace.ints64Bit_) { |
| 397 |
dbi->set_int_byte_size_api(Ioss::USE_INT64_API); |
| 398 |
} |
| 399 |
|
| 400 |
dbi->set_surface_split_type(Ioss::SPLIT_BY_DONT_SPLIT); |
| 401 |
dbi->set_field_separator(0); |
| 402 |
|
| 403 |
try { |
| 404 |
// NOTE: 'region' owns 'db' pointer at this time... |
| 405 |
Ioss::Region region(dbi, "region_1"); |
| 406 |
region.output_summary(std::cerr, true); |
| 407 |
|
| 408 |
if (dbi->int_byte_size_api() == 4) { |
| 409 |
progress("4-byte slice"); |
| 410 |
slice(region, nem_file, interFace, 1); |
| 411 |
} |
| 412 |
else { |
| 413 |
progress("8-byte slice"); |
| 414 |
slice(region, nem_file, interFace, static_cast<int64_t>(1)); |
| 415 |
} |
| 416 |
} |
| 417 |
catch (std::exception &e) { |
| 418 |
fmt::print(stderr, "\n{}\n\nSlice terminated due to exception\n", e.what()); |
| 419 |
exit(EXIT_FAILURE); |
| 420 |
} |
| 421 |
|
| 422 |
#ifdef SEACAS_HAVE_MPI |
| 423 |
MPI_Comm parentcomm; |
| 424 |
MPI_Comm_get_parent(&parentcomm); |
| 425 |
if (parentcomm != MPI_COMM_NULL) { |
| 426 |
int istatus = EXIT_SUCCESS; |
| 427 |
MPI_Send(&istatus, 1, MPI_INT, 0, 0, parentcomm); |
| 428 |
} |
| 429 |
MPI_Finalize(); |
| 430 |
#endif |
| 431 |
fmt::print(stderr, "\nHigh-Water Memory Use: {} bytes\n", |
| 432 |
fmt::group_digits(Ioss::MemoryUtils::get_hwm_memory_info())); |
| 433 |
fmt::print(stderr, "Total execution time = {:.5}\n", seacas_timer() - begin); |
| 434 |
fmt::print(stderr, "\nSlice execution successful.\n"); |
| 435 |
return EXIT_SUCCESS; |
| 436 |
} |
| 437 |
|
| 438 |
namespace { |
| 439 |
|
| 440 |
template <typename INT> |
| 441 |
void get_sidesets(const Ioss::Region ®ion, std::vector<Ioss::Region *> &proc_region, |
| 442 |
const std::vector<int> &elem_to_proc, INT /*dummy*/) |
| 443 |
{ |
| 444 |
progress(__func__); |
| 445 |
// This routine reads the sidesets in the global database; |
| 446 |
// and defines corresponding sidesets on each processor... |
| 447 |
size_t proc_count = proc_region.size(); |
| 448 |
|
| 449 |
const auto &ss = region.get_sidesets(); |
| 450 |
size_t set_count = ss.size(); |
| 451 |
|
| 452 |
for (size_t s = 0; s < set_count; s++) { |
| 453 |
auto *gss = ss[s]; |
| 454 |
const auto &ss_name = gss->name(); |
| 455 |
|
| 456 |
std::vector<Ioss::SideSet *> sset(proc_count); |
| 457 |
for (size_t p = 0; p < proc_count; p++) { |
| 458 |
sset[p] = new Ioss::SideSet(proc_region[p]->get_database(), ss_name); |
| 459 |
proc_region[p]->add(sset[p]); |
| 460 |
} |
| 461 |
|
| 462 |
const auto &side_blocks = gss->get_side_blocks(); |
| 463 |
for (const auto &gsb : side_blocks) { |
| 464 |
std::vector<INT> ss_elems; |
| 465 |
gsb->get_field_data("element_side_raw", ss_elems); |
| 466 |
|
| 467 |
std::vector<INT> pss(proc_count); |
| 468 |
for (size_t i = 0; i < ss_elems.size(); i += 2 /* elem,side pairs */) { |
| 469 |
int64_t elem = ss_elems[i] - 1; |
| 470 |
int p = elem_to_proc[elem]; |
| 471 |
pss[p]++; |
| 472 |
} |
| 473 |
|
| 474 |
const auto &name = gsb->name(); |
| 475 |
const auto &side_type = gsb->topology()->name(); |
| 476 |
const auto &elem_type = gsb->parent_element_topology()->name(); |
| 477 |
|
| 478 |
for (size_t p = 0; p < proc_count; p++) { |
| 479 |
auto *side_block = new Ioss::SideBlock(proc_region[p]->get_database(), name, side_type, |
| 480 |
elem_type, pss[p]); |
| 481 |
sset[p]->add(side_block); |
| 482 |
} |
| 483 |
} |
| 484 |
} |
| 485 |
} |
| 486 |
|
| 487 |
template <typename INT> |
| 488 |
void output_sidesets(const Ioss::Region ®ion, std::vector<Ioss::Region *> &proc_region, |
| 489 |
const std::vector<int> &elem_to_proc, size_t proc_begin, size_t proc_size, |
| 490 |
INT /*dummy*/) |
| 491 |
{ |
| 492 |
progress(__func__); |
| 493 |
// This routine reads the sidesets in the global database; |
| 494 |
// and outputs the sidesets on each processor... |
| 495 |
size_t proc_count = proc_region.size(); |
| 496 |
|
| 497 |
const auto &ss = region.get_sidesets(); |
| 498 |
size_t set_count = ss.size(); |
| 499 |
|
| 500 |
for (size_t s = 0; s < set_count; s++) { |
| 501 |
if (debug_level & 4) { |
| 502 |
progress("\tSideset " + std::to_string(s + 1)); |
| 503 |
} |
| 504 |
Ioss::SideSet *gss = ss[s]; |
| 505 |
const auto &ss_name = gss->name(); |
| 506 |
|
| 507 |
std::vector<Ioss::SideSet *> proc_ss(proc_count); |
| 508 |
for (size_t p = proc_begin; p < proc_begin + proc_size; p++) { |
| 509 |
proc_ss[p] = proc_region[p]->get_sideset(ss_name); |
| 510 |
} |
| 511 |
|
| 512 |
const auto &side_blocks = gss->get_side_blocks(); |
| 513 |
for (const auto &gsb : side_blocks) { |
| 514 |
const auto &sb_name = gsb->name(); |
| 515 |
|
| 516 |
std::vector<Ioss::SideBlock *> proc_sb(proc_count); |
| 517 |
std::vector<std::vector<INT>> psb_elems(proc_count); |
| 518 |
for (size_t p = proc_begin; p < proc_begin + proc_size; p++) { |
| 519 |
proc_sb[p] = proc_ss[p]->get_side_block(sb_name); |
| 520 |
size_t elem_count = proc_sb[p]->entity_count(); |
| 521 |
psb_elems[p].reserve(elem_count * 2); |
| 522 |
} |
| 523 |
|
| 524 |
std::vector<INT> ss_elems; |
| 525 |
gsb->get_field_data("element_side_raw", ss_elems); |
| 526 |
|
| 527 |
for (size_t i = 0; i < ss_elems.size(); i += 2 /* elem,side pairs */) { |
| 528 |
int64_t elem = ss_elems[i] - 1; |
| 529 |
int p = elem_to_proc[elem]; |
| 530 |
psb_elems[p].push_back(elem + 1); |
| 531 |
psb_elems[p].push_back(ss_elems[i + 1]); |
| 532 |
} |
| 533 |
|
| 534 |
for (size_t p = proc_begin; p < proc_begin + proc_size; p++) { |
| 535 |
Ioss::SideBlock *psb = proc_sb[p]; |
| 536 |
psb->put_field_data("element_side", psb_elems[p]); |
| 537 |
proc_progress(p, proc_count); |
| 538 |
} |
| 539 |
} |
| 540 |
} |
| 541 |
if (set_count > 0) { |
| 542 |
static bool output = false; |
| 543 |
if (!output) { |
| 544 |
fmt::print(stderr, "WARNING: Sideset distribution factors not yet handled correctly.\n"); |
| 545 |
output = true; |
| 546 |
} |
| 547 |
} |
| 548 |
} |
| 549 |
|
| 550 |
template <typename INT> |
| 551 |
void output_communication_map(const Ioss::Region &global_region, |
| 552 |
std::vector<Ioss::Region *> &proc_region, |
| 553 |
const std::vector<int> &node_to_proc, |
| 554 |
const std::vector<INT> &node_to_proc_pointer, size_t proc_begin, |
| 555 |
size_t proc_size) |
| 556 |
{ |
| 557 |
progress(__func__); |
| 558 |
|
| 559 |
std::vector<std::vector<INT>> border_node_proc_map(proc_size); |
| 560 |
|
| 561 |
INT global_node_count = global_region.get_property("node_count").get_int(); |
| 562 |
// Iterate all nodes and count the number of processors it is on: |
| 563 |
for (INT i = 0; i < global_node_count; i++) { |
| 564 |
size_t node_proc_count = node_to_proc_pointer[i + 1] - node_to_proc_pointer[i]; |
| 565 |
if (node_proc_count > 1) { |
| 566 |
// Get the <node,proc> pairs for all border nodes on this processor... |
| 567 |
// Not efficient at this time... |
| 568 |
size_t beg = node_to_proc_pointer[i]; |
| 569 |
size_t end = node_to_proc_pointer[i + 1]; |
| 570 |
for (size_t j = beg; j < end; j++) { |
| 571 |
size_t node = i + 1; |
| 572 |
size_t proc = node_to_proc[j]; |
| 573 |
for (size_t k = beg; k < end; k++) { |
| 574 |
if (j == k) { |
| 575 |
continue; |
| 576 |
} |
| 577 |
size_t p = node_to_proc[k]; |
| 578 |
if (p >= proc_begin && p < proc_begin + proc_size) { |
| 579 |
border_node_proc_map[p - proc_begin].push_back(node); |
| 580 |
border_node_proc_map[p - proc_begin].push_back(proc); |
| 581 |
} |
| 582 |
} |
| 583 |
} |
| 584 |
} |
| 585 |
} |
| 586 |
|
| 587 |
progress("border_node_proc_map fully populated"); |
| 588 |
size_t proc_count = proc_region.size(); |
| 589 |
for (size_t p = proc_begin; p < proc_begin + proc_size; p++) { |
| 590 |
const auto &commset = proc_region[p]->get_commsets()[0]; |
| 591 |
commset->put_field_data("entity_processor", border_node_proc_map[p - proc_begin]); |
| 592 |
border_node_proc_map[p - proc_begin].clear(); |
| 593 |
proc_progress(p, proc_count); |
| 594 |
} |
| 595 |
} |
| 596 |
|
| 597 |
template <typename INT> |
| 598 |
void define_communication_data(const Ioss::Region &global_region, |
| 599 |
std::vector<Ioss::Region *> &proc_region, |
| 600 |
const std::vector<int> &node_to_proc, |
| 601 |
const std::vector<INT> &node_to_proc_pointer) |
| 602 |
{ |
| 603 |
progress(__func__); |
| 604 |
// This routine categorizes the nodes on a processor as interior |
| 605 |
// or border. |
| 606 |
// TODO(gdsjaar): Categorize elements also. For now, all treated as |
| 607 |
// interior which works for sierra-based applications |
| 608 |
|
| 609 |
// The node_to_proc_pointer has information about the number of |
| 610 |
// processors that a node is shared with. If the count is 1, then |
| 611 |
// the node is interior; otherwise, it is border. |
| 612 |
|
| 613 |
// Allocates: |
| 614 |
// * interior_nodes INT size - #interior nodes |
| 615 |
// * border_nodes_proc_map INT size - (proc-node) pair for each border node |
| 616 |
|
| 617 |
INT global_node_count = global_region.get_property("node_count").get_int(); |
| 618 |
size_t proc_count = proc_region.size(); |
| 619 |
std::vector<INT> interior_nodes(proc_count); |
| 620 |
std::vector<INT> border_nodes(proc_count); |
| 621 |
|
| 622 |
// Iterate all nodes and count the number of processors it is on: |
| 623 |
for (INT i = 0; i < global_node_count; i++) { |
| 624 |
size_t node_proc_count = node_to_proc_pointer[i + 1] - node_to_proc_pointer[i]; |
| 625 |
if (node_proc_count == 1) { |
| 626 |
size_t proc = node_to_proc[node_to_proc_pointer[i]]; |
| 627 |
interior_nodes[proc]++; |
| 628 |
} |
| 629 |
else { |
| 630 |
// Get the <node,proc> pairs for all border nodes on this processor... |
| 631 |
// Not efficient at this time... |
| 632 |
size_t beg = node_to_proc_pointer[i]; |
| 633 |
size_t end = node_to_proc_pointer[i + 1]; |
| 634 |
for (size_t j = beg; j < end; j++) { |
| 635 |
for (size_t k = beg; k < end; k++) { |
| 636 |
if (j == k) { |
| 637 |
continue; |
| 638 |
} |
| 639 |
size_t p = node_to_proc[k]; |
| 640 |
border_nodes[p]++; |
| 641 |
} |
| 642 |
} |
| 643 |
} |
| 644 |
} |
| 645 |
|
| 646 |
INT global_element_count = global_region.get_property("element_count").get_int(); |
| 647 |
|
| 648 |
// Categorize each element as interior... |
| 649 |
// Categorize the remaining nodes as border... |
| 650 |
for (size_t p = 0; p < proc_count; p++) { |
| 651 |
Ioss::Region *region = proc_region[p]; |
| 652 |
|
| 653 |
INT element_count = region->get_property("element_count").get_int(); |
| 654 |
INT node_count = region->get_property("node_count").get_int(); |
| 655 |
INT border_node_cnt = node_count - interior_nodes[p]; |
| 656 |
|
| 657 |
region->property_add(Ioss::Property("global_node_count", global_node_count)); |
| 658 |
region->property_add(Ioss::Property("global_element_count", global_element_count)); |
| 659 |
region->property_add(Ioss::Property("processor_count", static_cast<int>(proc_count))); |
| 660 |
region->property_add(Ioss::Property("my_processor", static_cast<int>(p))); |
| 661 |
|
| 662 |
region->property_add(Ioss::Property("internal_node_count", interior_nodes[p])); |
| 663 |
region->property_add(Ioss::Property("border_node_count", border_node_cnt)); |
| 664 |
region->property_add(Ioss::Property("internal_element_count", element_count)); |
| 665 |
region->property_add(Ioss::Property("border_element_count", 0)); |
| 666 |
|
| 667 |
// Add commset data... The length of the commset is the number |
| 668 |
// of <node,proc> pairs for all border nodes. |
| 669 |
// |
| 670 |
// For each node on this processor that isn't an interior node, |
| 671 |
// create the <node,proc> pair... |
| 672 |
auto *commset = |
| 673 |
new Ioss::CommSet(region->get_database(), "commset_node", "node", border_nodes[p]); |
| 674 |
commset->property_add(Ioss::Property("id", 1)); |
| 675 |
region->add(commset); |
| 676 |
if (debug_level & 2) { |
| 677 |
fmt::print(stderr, "Commset for processor {} has {} entries.\n", p, border_nodes[p]); |
| 678 |
} |
| 679 |
} |
| 680 |
} |
| 681 |
|
| 682 |
template <typename INT> |
| 683 |
void get_nodesets(const Ioss::Region ®ion, std::vector<Ioss::Region *> &proc_region, |
| 684 |
const std::vector<int> &node_to_proc, |
| 685 |
const std::vector<INT> &node_to_proc_pointer) |
| 686 |
{ |
| 687 |
progress(__func__); |
| 688 |
// This routine reads the nodesets in the global database; |
| 689 |
// and defines corresponding nodesets on each processor... |
| 690 |
size_t proc_count = proc_region.size(); |
| 691 |
|
| 692 |
const auto &ns = region.get_nodesets(); |
| 693 |
size_t set_count = ns.size(); |
| 694 |
|
| 695 |
for (size_t s = 0; s < set_count; s++) { |
| 696 |
std::vector<INT> pns(proc_count); |
| 697 |
Ioss::NodeSet *gns = ns[s]; |
| 698 |
|
| 699 |
std::vector<INT> ns_nodes; |
| 700 |
gns->get_field_data("ids_raw", ns_nodes); |
| 701 |
|
| 702 |
for (size_t i = 0; i < ns_nodes.size(); i++) { |
| 703 |
int64_t node = ns_nodes[i] - 1; |
| 704 |
size_t p_beg = node_to_proc_pointer[node]; |
| 705 |
size_t p_end = node_to_proc_pointer[node + 1]; |
| 706 |
for (size_t j = p_beg; j < p_end; j++) { |
| 707 |
size_t p = node_to_proc[j]; |
| 708 |
pns[p]++; |
| 709 |
} |
| 710 |
} |
| 711 |
|
| 712 |
const auto &name = ns[s]->name(); |
| 713 |
if (debug_level & 2) { |
| 714 |
fmt::print(stderr, "\tNodeset {}--", name); |
| 715 |
} |
| 716 |
for (size_t p = 0; p < proc_count; p++) { |
| 717 |
auto *node_set = new Ioss::NodeSet(proc_region[p]->get_database(), name, pns[p]); |
| 718 |
proc_region[p]->add(node_set); |
| 719 |
if (debug_level & 2) { |
| 720 |
fmt::print(stderr, "{}:{}, ", p, pns[p]); |
| 721 |
} |
| 722 |
} |
| 723 |
if (debug_level & 2) { |
| 724 |
fmt::print(stderr, "\n"); |
| 725 |
} |
| 726 |
} |
| 727 |
} |
| 728 |
|
| 729 |
template <typename INT> |
| 730 |
void output_nodesets(const Ioss::Region ®ion, std::vector<Ioss::Region *> &proc_region, |
| 731 |
const std::vector<int> &node_to_proc, |
| 732 |
const std::vector<INT> &node_to_proc_pointer, size_t proc_begin, |
| 733 |
size_t proc_size) |
| 734 |
{ |
| 735 |
progress(__func__); |
| 736 |
// This routine reads the nodesets in the global database; |
| 737 |
// and defines corresponding nodesets on each processor... |
| 738 |
size_t proc_count = proc_region.size(); |
| 739 |
|
| 740 |
const auto &ns = region.get_nodesets(); |
| 741 |
size_t set_count = ns.size(); |
| 742 |
|
| 743 |
for (size_t s = 0; s < set_count; s++) { |
| 744 |
if (debug_level & 4) { |
| 745 |
progress("\tNodeSet " + std::to_string(s + 1)); |
| 746 |
} |
| 747 |
Ioss::NodeSet *gns = ns[s]; |
| 748 |
|
| 749 |
std::vector<INT> ns_nodes; |
| 750 |
gns->get_field_data("ids_raw", ns_nodes); |
| 751 |
|
| 752 |
std::vector<double> ns_df; |
| 753 |
gns->get_field_data("distribution_factors", ns_df); |
| 754 |
|
| 755 |
std::vector<std::vector<INT>> pns_nodes(proc_count); |
| 756 |
std::vector<std::vector<double>> pns_df(proc_count); |
| 757 |
for (size_t p = proc_begin; p < proc_begin + proc_size; p++) { |
| 758 |
size_t node_count = proc_region[p]->get_nodesets()[s]->entity_count(); |
| 759 |
pns_nodes[p].reserve(node_count); |
| 760 |
pns_df[p].reserve(node_count); |
| 761 |
} |
| 762 |
|
| 763 |
for (size_t i = 0; i < ns_nodes.size(); i++) { |
| 764 |
int64_t node = ns_nodes[i] - 1; |
| 765 |
size_t p_beg = node_to_proc_pointer[node]; |
| 766 |
size_t p_end = node_to_proc_pointer[node + 1]; |
| 767 |
for (size_t j = p_beg; j < p_end; j++) { |
| 768 |
size_t p = node_to_proc[j]; |
| 769 |
pns_nodes[p].push_back(node + 1); |
| 770 |
pns_df[p].push_back(ns_df[i]); |
| 771 |
} |
| 772 |
} |
| 773 |
|
| 774 |
for (size_t p = proc_begin; p < proc_begin + proc_size; p++) { |
| 775 |
Ioss::NodeSet *proc_ns = proc_region[p]->get_nodesets()[s]; |
| 776 |
proc_ns->put_field_data("ids", pns_nodes[p]); |
| 777 |
proc_ns->put_field_data("distribution_factors", pns_df[p]); |
| 778 |
proc_progress(p, proc_count); |
| 779 |
} |
| 780 |
} |
| 781 |
} |
| 782 |
|
| 783 |
template <typename INT> |
| 784 |
void output_node_map(const Ioss::Region ®ion, std::vector<Ioss::Region *> &proc_region, |
| 785 |
const std::vector<int> &node_to_proc, |
| 786 |
const std::vector<INT> &node_to_proc_pointer, size_t proc_begin, |
| 787 |
size_t proc_size) |
| 788 |
{ |
| 789 |
progress(__func__); |
| 790 |
// This is the processor-local to global-implicit node map... |
| 791 |
// This maps the 1..#node in the global mesh to each processor... |
| 792 |
size_t node_count = region.get_property("node_count").get_int(); |
| 793 |
size_t proc_count = proc_region.size(); |
| 794 |
|
| 795 |
std::vector<std::vector<INT>> proc_map(proc_count); |
| 796 |
for (size_t p = proc_begin; p < proc_begin + proc_size; p++) { |
| 797 |
size_t pnode_count = proc_region[p]->get_property("node_count").get_int(); |
| 798 |
proc_map[p].reserve(pnode_count); |
| 799 |
} |
| 800 |
|
| 801 |
for (size_t i = 0; i < node_count; i++) { |
| 802 |
size_t p_beg = node_to_proc_pointer[i]; |
| 803 |
size_t p_end = node_to_proc_pointer[i + 1]; |
| 804 |
for (size_t j = p_beg; j < p_end; j++) { |
| 805 |
size_t p = node_to_proc[j]; |
| 806 |
if (p >= proc_begin && p < proc_begin + proc_size) { |
| 807 |
proc_map[p].push_back(i + 1); |
| 808 |
} |
| 809 |
} |
| 810 |
} |
| 811 |
|
| 812 |
for (size_t p = proc_begin; p < proc_begin + proc_size; p++) { |
| 813 |
Ioss::NodeBlock *nb = proc_region[p]->get_node_blocks()[0]; |
| 814 |
nb->put_field_data("ids", proc_map[p]); |
| 815 |
proc_map[p].clear(); |
| 816 |
proc_progress(p, proc_count); |
| 817 |
} |
| 818 |
} |
| 819 |
|
| 820 |
template <typename INT> |
| 821 |
void output_global_node_map(const Ioss::Region ®ion, std::vector<Ioss::Region *> &proc_region, |
| 822 |
const std::vector<int> &node_to_proc, |
| 823 |
const std::vector<INT> &node_to_proc_pointer, size_t proc_begin, |
| 824 |
size_t proc_size) |
| 825 |
{ |
| 826 |
progress(__func__); |
| 827 |
// This is the processor-local to global-implicit node map... |
| 828 |
// This maps the node_number map (if it exists) in the global mesh |
| 829 |
// to each processor... |
| 830 |
std::vector<INT> ids; |
| 831 |
Ioss::NodeBlock *gnb = region.get_node_blocks()[0]; |
| 832 |
gnb->get_field_data("ids", ids); |
| 833 |
|
| 834 |
// Check whether the map is sequential (X maps to X); |
| 835 |
bool sequential = is_sequential(ids); |
| 836 |
if (!sequential) { |
| 837 |
fmt::print(stderr, "Node map is not sequential...\n"); |
| 838 |
} |
| 839 |
|
| 840 |
size_t node_count = region.get_property("node_count").get_int(); |
| 841 |
size_t proc_count = proc_region.size(); |
| 842 |
|
| 843 |
std::vector<std::vector<INT>> proc_map(proc_count); |
| 844 |
for (size_t p = proc_begin; p < proc_begin + proc_size; p++) { |
| 845 |
size_t pnode_count = proc_region[p]->get_property("node_count").get_int(); |
| 846 |
proc_map[p].reserve(pnode_count); |
| 847 |
} |
| 848 |
|
| 849 |
for (size_t i = 0; i < node_count; i++) { |
| 850 |
size_t p_beg = node_to_proc_pointer[i]; |
| 851 |
size_t p_end = node_to_proc_pointer[i + 1]; |
| 852 |
for (size_t j = p_beg; j < p_end; j++) { |
| 853 |
size_t p = node_to_proc[j]; |
| 854 |
if (p >= proc_begin && p < proc_begin + proc_size) { |
| 855 |
proc_map[p].push_back(ids[i]); |
| 856 |
} |
| 857 |
} |
| 858 |
} |
| 859 |
|
| 860 |
for (size_t p = proc_begin; p < proc_begin + proc_size; p++) { |
| 861 |
Ioss::NodeBlock *nb = proc_region[p]->get_node_blocks()[0]; |
| 862 |
nb->put_field_data("ids", proc_map[p]); |
| 863 |
proc_map[p].clear(); |
| 864 |
proc_progress(p, proc_count); |
| 865 |
} |
| 866 |
} |
| 867 |
|
| 868 |
template <typename INT> |
| 869 |
void output_element_map(const Ioss::Region ®ion, std::vector<Ioss::Region *> &proc_region, |
| 870 |
const std::vector<int> &elem_to_proc, size_t proc_begin, size_t proc_size, |
| 871 |
INT /* dummy */) |
| 872 |
{ |
| 873 |
progress(__func__); |
| 874 |
// map[p][b] = map for block b on processor p |
| 875 |
size_t proc_count = proc_region.size(); |
| 876 |
|
| 877 |
const auto &ebs = region.get_element_blocks(); |
| 878 |
size_t block_count = ebs.size(); |
| 879 |
|
| 880 |
size_t offset = 0; |
| 881 |
for (size_t b = 0; b < block_count; b++) { |
| 882 |
if (debug_level & 4) { |
| 883 |
progress("\tBlock " + std::to_string(b + 1)); |
| 884 |
} |
| 885 |
#if 0 |
| 886 |
std::vector<INT> ids; |
| 887 |
ebs[b]->get_field_data("ids", ids); |
| 888 |
#endif |
| 889 |
|
| 890 |
std::vector<std::vector<INT>> map(proc_count); |
| 891 |
for (size_t p = proc_begin; p < proc_begin + proc_size; p++) { |
| 892 |
const auto &proc_ebs = proc_region[p]->get_element_blocks(); |
| 893 |
size_t proc_element_count = proc_ebs[b]->entity_count(); |
| 894 |
map[p].reserve(proc_element_count); |
| 895 |
} |
| 896 |
|
| 897 |
size_t element_count = ebs[b]->entity_count(); |
| 898 |
|
| 899 |
for (size_t j = 0; j < element_count; j++) { |
| 900 |
size_t p = elem_to_proc[offset + j]; |
| 901 |
if (p >= proc_begin && p < proc_begin + proc_size) { |
| 902 |
#if 0 |
| 903 |
map[p].push_back(ids[j]); |
| 904 |
#else |
| 905 |
map[p].push_back(offset + j + 1); |
| 906 |
#endif |
| 907 |
} |
| 908 |
} |
| 909 |
offset += element_count; |
| 910 |
|
| 911 |
for (size_t p = proc_begin; p < proc_begin + proc_size; p++) { |
| 912 |
const auto &proc_ebs = proc_region[p]->get_element_blocks(); |
| 913 |
proc_ebs[b]->put_field_data("ids", map[p]); |
| 914 |
map[p].clear(); |
| 915 |
proc_progress(p, proc_count); |
| 916 |
} |
| 917 |
} |
| 918 |
} |
| 919 |
|
| 920 |
template <typename INT> |
| 921 |
void output_coordinates(const Ioss::Region ®ion, std::vector<Ioss::Region *> &proc_region, |
| 922 |
const std::vector<int> &node_to_proc, |
| 923 |
const std::vector<INT> &node_to_proc_pointer, size_t proc_begin, |
| 924 |
size_t proc_size) |
| 925 |
{ |
| 926 |
progress(__func__); |
| 927 |
size_t spatial_dimension = region.get_property("spatial_dimension").get_int(); |
| 928 |
|
| 929 |
std::vector<double> glob_coord_x; |
| 930 |
std::vector<double> glob_coord_y; |
| 931 |
std::vector<double> glob_coord_z; |
| 932 |
Ioss::NodeBlock *gnb = region.get_node_blocks()[0]; |
| 933 |
|
| 934 |
// Distribute nodal coordinates to each processor... |
| 935 |
// coordinates[p][i] = x,y,z coordinates on processor p |
| 936 |
size_t processor_count = proc_region.size(); |
| 937 |
std::vector<std::vector<double>> coordinates_x(processor_count); |
| 938 |
std::vector<std::vector<double>> coordinates_y(processor_count); |
| 939 |
std::vector<std::vector<double>> coordinates_z(processor_count); |
| 940 |
for (size_t p = proc_begin; p < proc_begin + proc_size; p++) { |
| 941 |
size_t pnode_count = proc_region[p]->get_property("node_count").get_int(); |
| 942 |
coordinates_x[p].reserve(pnode_count); |
| 943 |
coordinates_y[p].reserve(pnode_count); |
| 944 |
coordinates_z[p].reserve(pnode_count); |
| 945 |
} |
| 946 |
progress("\tReserve processor coordinate vectors"); |
| 947 |
|
| 948 |
Ioss::DatabaseIO *db = region.get_database(); |
| 949 |
|
| 950 |
size_t node_count = region.get_property("node_count").get_int(); |
| 951 |
|
| 952 |
if (node_count > partial_count) { |
| 953 |
int exoid = db->get_file_pointer(); |
| 954 |
|
| 955 |
glob_coord_x.resize(partial_count); |
| 956 |
glob_coord_y.resize(partial_count); |
| 957 |
glob_coord_z.resize(partial_count); |
| 958 |
for (size_t beg = 1; beg <= node_count; beg += partial_count) { |
| 959 |
size_t count = partial_count; |
| 960 |
if (beg + count - 1 > node_count) { |
| 961 |
count = node_count - beg + 1; |
| 962 |
} |
| 963 |
|
| 964 |
ex_get_partial_coord(exoid, beg, count, Data(glob_coord_x), Data(glob_coord_y), |
| 965 |
Data(glob_coord_z)); |
| 966 |
progress("\tpartial_coord: " + std::to_string(beg) + " " + std::to_string(count)); |
| 967 |
|
| 968 |
for (size_t i = 0; i < count; i++) { |
| 969 |
size_t ii = beg + i - 1; |
| 970 |
size_t p_beg = node_to_proc_pointer[ii]; |
| 971 |
size_t p_end = node_to_proc_pointer[ii + 1]; |
| 972 |
for (size_t j = p_beg; j < p_end; j++) { |
| 973 |
size_t p = node_to_proc[j]; |
| 974 |
if (p >= proc_begin && p < proc_begin + proc_size) { |
| 975 |
coordinates_x[p].push_back(glob_coord_x[i]); |
| 976 |
coordinates_y[p].push_back(glob_coord_y[i]); |
| 977 |
coordinates_z[p].push_back(glob_coord_z[i]); |
| 978 |
} |
| 979 |
} |
| 980 |
} |
| 981 |
} |
| 982 |
} |
| 983 |
else { |
| 984 |
gnb->get_field_data("mesh_model_coordinates_x", glob_coord_x); |
| 985 |
if (spatial_dimension > 1) { |
| 986 |
gnb->get_field_data("mesh_model_coordinates_y", glob_coord_y); |
| 987 |
} |
| 988 |
if (spatial_dimension > 2) { |
| 989 |
gnb->get_field_data("mesh_model_coordinates_z", glob_coord_z); |
| 990 |
} |
| 991 |
progress("\tRead global mesh_model_coordinates"); |
| 992 |
|
| 993 |
for (size_t i = 0; i < node_count; i++) { |
| 994 |
size_t p_beg = node_to_proc_pointer[i]; |
| 995 |
size_t p_end = node_to_proc_pointer[i + 1]; |
| 996 |
for (size_t j = p_beg; j < p_end; j++) { |
| 997 |
size_t p = node_to_proc[j]; |
| 998 |
if (p >= proc_begin && p < proc_begin + proc_size) { |
| 999 |
coordinates_x[p].push_back(glob_coord_x[i]); |
| 1000 |
coordinates_y[p].push_back(glob_coord_y[i]); |
| 1001 |
coordinates_z[p].push_back(glob_coord_z[i]); |
| 1002 |
} |
| 1003 |
} |
| 1004 |
} |
| 1005 |
} |
| 1006 |
progress("\tPopulate processor coordinate vectors"); |
| 1007 |
Ioss::Utils::clear(glob_coord_x); |
| 1008 |
Ioss::Utils::clear(glob_coord_y); |
| 1009 |
Ioss::Utils::clear(glob_coord_z); |
| 1010 |
|
| 1011 |
for (size_t p = proc_begin; p < proc_begin + proc_size; p++) { |
| 1012 |
Ioss::NodeBlock *nb = proc_region[p]->get_node_blocks()[0]; |
| 1013 |
nb->put_field_data("mesh_model_coordinates_x", coordinates_x[p]); |
| 1014 |
if (spatial_dimension > 1) { |
| 1015 |
nb->put_field_data("mesh_model_coordinates_y", coordinates_y[p]); |
| 1016 |
} |
| 1017 |
if (spatial_dimension > 2) { |
| 1018 |
nb->put_field_data("mesh_model_coordinates_z", coordinates_z[p]); |
| 1019 |
} |
| 1020 |
proc_progress(p, processor_count); |
| 1021 |
} |
| 1022 |
progress("\tOutput processor coordinate vectors"); |
| 1023 |
} |
| 1024 |
|
| 1025 |
// Output a component at a time... |
| 1026 |
template <typename INT> |
| 1027 |
void output_coordinates_c(const Ioss::Region ®ion, std::vector<Ioss::Region *> &proc_region, |
| 1028 |
const std::vector<int> &node_to_proc, |
| 1029 |
const std::vector<INT> &node_to_proc_pointer, size_t proc_begin, |
| 1030 |
size_t proc_size) |
| 1031 |
{ |
| 1032 |
progress(__func__); |
| 1033 |
std::vector<double> glob_coord; |
| 1034 |
Ioss::NodeBlock *gnb = region.get_node_blocks()[0]; |
| 1035 |
|
| 1036 |
std::array<std::string, 3> field_name{"mesh_model_coordinates_x", "mesh_model_coordinates_y", |
| 1037 |
"mesh_model_coordinates_z"}; |
| 1038 |
// Distribute nodal coordinates to each processor... |
| 1039 |
// coordinates[p][i] = x,y,z coordinates on processor p |
| 1040 |
size_t processor_count = proc_region.size(); |
| 1041 |
std::vector<std::vector<double>> coordinates(processor_count); |
| 1042 |
|
| 1043 |
Ioss::DatabaseIO *db = region.get_database(); |
| 1044 |
|
| 1045 |
size_t node_count = region.get_property("node_count").get_int(); |
| 1046 |
|
| 1047 |
for (size_t p = proc_begin; p < proc_begin + proc_size; p++) { |
| 1048 |
size_t pnode_count = proc_region[p]->get_property("node_count").get_int(); |
| 1049 |
coordinates[p].reserve(pnode_count); |
| 1050 |
} |
| 1051 |
progress("\tReserve processor coordinate vectors"); |
| 1052 |
|
| 1053 |
size_t spatial_dimension = region.get_property("spatial_dimension").get_int(); |
| 1054 |
|
| 1055 |
for (size_t comp = 0; comp < spatial_dimension; comp++) { |
| 1056 |
for (size_t p = proc_begin; p < proc_begin + proc_size; p++) { |
| 1057 |
coordinates[p].resize(0); |
| 1058 |
} |
| 1059 |
|
| 1060 |
if (node_count > partial_count) { |
| 1061 |
int exoid = db->get_file_pointer(); |
| 1062 |
|
| 1063 |
glob_coord.resize(partial_count); |
| 1064 |
for (size_t beg = 1; beg <= node_count; beg += partial_count) { |
| 1065 |
size_t count = partial_count; |
| 1066 |
if (beg + count - 1 > node_count) { |
| 1067 |
count = node_count - beg + 1; |
| 1068 |
} |
| 1069 |
|
| 1070 |
switch (comp) { |
| 1071 |
case 0: |
| 1072 |
ex_get_partial_coord(exoid, beg, count, Data(glob_coord), nullptr, nullptr); |
| 1073 |
break; |
| 1074 |
case 1: |
| 1075 |
ex_get_partial_coord(exoid, beg, count, nullptr, Data(glob_coord), nullptr); |
| 1076 |
break; |
| 1077 |
case 2: |
| 1078 |
ex_get_partial_coord(exoid, beg, count, nullptr, nullptr, Data(glob_coord)); |
| 1079 |
break; |
| 1080 |
} |
| 1081 |
progress("\tpartial_coord: " + std::to_string(beg) + " " + std::to_string(count)); |
| 1082 |
|
| 1083 |
for (size_t i = 0; i < count; i++) { |
| 1084 |
size_t ii = beg + i - 1; |
| 1085 |
size_t p_beg = node_to_proc_pointer[ii]; |
| 1086 |
size_t p_end = node_to_proc_pointer[ii + 1]; |
| 1087 |
for (size_t j = p_beg; j < p_end; j++) { |
| 1088 |
size_t p = node_to_proc[j]; |
| 1089 |
if (p >= proc_begin && p < proc_begin + proc_size) { |
| 1090 |
coordinates[p].push_back(glob_coord[i]); |
| 1091 |
} |
| 1092 |
} |
| 1093 |
} |
| 1094 |
} |
| 1095 |
} |
| 1096 |
else { |
| 1097 |
gnb->get_field_data(field_name[comp], glob_coord); |
| 1098 |
progress("\tRead global mesh_model_coordinates"); |
| 1099 |
|
| 1100 |
for (size_t i = 0; i < node_count; i++) { |
| 1101 |
size_t p_beg = node_to_proc_pointer[i]; |
| 1102 |
size_t p_end = node_to_proc_pointer[i + 1]; |
| 1103 |
for (size_t j = p_beg; j < p_end; j++) { |
| 1104 |
size_t p = node_to_proc[j]; |
| 1105 |
if (p >= proc_begin && p < proc_begin + proc_size) { |
| 1106 |
coordinates[p].push_back(glob_coord[i]); |
| 1107 |
} |
| 1108 |
} |
| 1109 |
} |
| 1110 |
} |
| 1111 |
progress("\tPopulate processor coordinate vectors"); |
| 1112 |
Ioss::Utils::clear(glob_coord); |
| 1113 |
|
| 1114 |
for (size_t p = proc_begin; p < proc_begin + proc_size; p++) { |
| 1115 |
Ioss::NodeBlock *nb = proc_region[p]->get_node_blocks()[0]; |
| 1116 |
nb->put_field_data(field_name[comp], coordinates[p]); |
| 1117 |
proc_progress(p, processor_count); |
| 1118 |
} |
| 1119 |
} |
| 1120 |
progress("\tOutput processor coordinate vectors"); |
| 1121 |
} |
| 1122 |
|
| 1123 |
template <typename INT> |
| 1124 |
void output_connectivity(const Ioss::Region ®ion, std::vector<Ioss::Region *> &proc_region, |
| 1125 |
const std::vector<int> &elem_to_proc, size_t proc_begin, |
| 1126 |
size_t proc_size, INT /*dummy*/) |
| 1127 |
{ |
| 1128 |
// Read connectivity and partition to each processor/block. |
| 1129 |
// connectvity[p][b] = connectivity for block b on processor p |
| 1130 |
|
| 1131 |
progress(__func__); |
| 1132 |
const auto &ebs = region.get_element_blocks(); |
| 1133 |
size_t block_count = ebs.size(); |
| 1134 |
|
| 1135 |
size_t processor_count = proc_region.size(); |
| 1136 |
|
| 1137 |
Ioss::DatabaseIO *db = region.get_database(); |
| 1138 |
|
| 1139 |
std::vector<INT> glob_conn; |
| 1140 |
size_t offset = 0; |
| 1141 |
|
| 1142 |
for (size_t b = 0; b < block_count; b++) { |
| 1143 |
std::vector<std::vector<INT>> connectivity(processor_count); |
| 1144 |
size_t element_count = ebs[b]->entity_count(); |
| 1145 |
size_t element_nodes = ebs[b]->topology()->number_nodes(); |
| 1146 |
size_t block_id = ebs[b]->get_property("id").get_int(); |
| 1147 |
|
| 1148 |
for (size_t p = proc_begin; p < proc_begin + proc_size; p++) { |
| 1149 |
const auto &pebs = proc_region[p]->get_element_blocks(); |
| 1150 |
size_t pelement_count = pebs[b]->entity_count(); |
| 1151 |
size_t pelement_nodes = pebs[b]->topology()->number_nodes(); |
| 1152 |
connectivity[p].reserve(pelement_count * pelement_nodes); // Use reserve, not resize |
| 1153 |
} |
| 1154 |
|
| 1155 |
// Do a 'partial_count' elements at a time... |
| 1156 |
if (element_count >= partial_count) { |
| 1157 |
int exoid = db->get_file_pointer(); |
| 1158 |
|
| 1159 |
glob_conn.resize(partial_count * element_nodes); |
| 1160 |
for (size_t beg = 1; beg <= element_count; beg += partial_count) { |
| 1161 |
size_t count = partial_count; |
| 1162 |
if (beg + count - 1 > element_count) { |
| 1163 |
count = element_count - beg + 1; |
| 1164 |
} |
| 1165 |
|
| 1166 |
ex_get_partial_conn(exoid, EX_ELEM_BLOCK, block_id, beg, count, Data(glob_conn), nullptr, |
| 1167 |
nullptr); |
| 1168 |
progress(fmt::format("\tpartial_conn-- start: {}\tcount: {}", fmt::group_digits(beg), |
| 1169 |
fmt::group_digits(count))); |
| 1170 |
|
| 1171 |
size_t el = 0; |
| 1172 |
for (size_t j = 0; j < count; j++) { |
| 1173 |
size_t p = elem_to_proc[offset + j]; |
| 1174 |
if (p >= proc_begin && p < proc_begin + proc_size) { |
| 1175 |
for (size_t k = 0; k < element_nodes; k++) { |
| 1176 |
connectivity[p].push_back(glob_conn[el++]); |
| 1177 |
} |
| 1178 |
} |
| 1179 |
else { |
| 1180 |
el += element_nodes; |
| 1181 |
} |
| 1182 |
} |
| 1183 |
offset += count; |
| 1184 |
} |
| 1185 |
} |
| 1186 |
else { |
| 1187 |
ebs[b]->get_field_data("connectivity_raw", glob_conn); |
| 1188 |
|
| 1189 |
size_t el = 0; |
| 1190 |
for (size_t j = 0; j < element_count; j++) { |
| 1191 |
size_t p = elem_to_proc[offset + j]; |
| 1192 |
if (p >= proc_begin && p < proc_begin + proc_size) { |
| 1193 |
for (size_t k = 0; k < element_nodes; k++) { |
| 1194 |
connectivity[p].push_back(glob_conn[el++]); |
| 1195 |
} |
| 1196 |
} |
| 1197 |
else { |
| 1198 |
el += element_nodes; |
| 1199 |
} |
| 1200 |
} |
| 1201 |
offset += element_count; |
| 1202 |
} |
| 1203 |
|
| 1204 |
for (size_t p = proc_begin; p < proc_begin + proc_size; p++) { |
| 1205 |
const auto &proc_ebs = proc_region[p]->get_element_blocks(); |
| 1206 |
Ioss::ElementBlock *eb = proc_ebs[b]; |
| 1207 |
eb->put_field_data("connectivity", connectivity[p]); |
| 1208 |
} |
| 1209 |
} |
| 1210 |
} |
| 1211 |
|
| 1212 |
template <typename INT> |
| 1213 |
void get_proc_elem_block_count(const Ioss::Region ®ion, std::vector<int> &elem_to_proc, |
| 1214 |
std::vector<std::vector<INT>> &proc_elem_block_cnt) |
| 1215 |
{ |
| 1216 |
progress(__func__); |
| 1217 |
const auto &ebs = region.get_element_blocks(); |
| 1218 |
size_t block_count = ebs.size(); |
| 1219 |
size_t begin = 0; |
| 1220 |
for (size_t i = 0; i < block_count; i++) { |
| 1221 |
size_t end = begin + ebs[i]->entity_count(); |
| 1222 |
for (size_t j = begin; j < end; j++) { |
| 1223 |
size_t processor = elem_to_proc[j]; |
| 1224 |
proc_elem_block_cnt[i][processor]++; |
| 1225 |
} |
| 1226 |
begin = end; |
| 1227 |
} |
| 1228 |
|
| 1229 |
size_t processor_count = proc_elem_block_cnt[0].size(); |
| 1230 |
for (size_t i = 0; i < processor_count; i++) { |
| 1231 |
size_t sum = 0; |
| 1232 |
for (size_t j = 0; j < block_count; j++) { |
| 1233 |
sum += proc_elem_block_cnt[j][i]; |
| 1234 |
} |
| 1235 |
proc_elem_block_cnt[block_count][i] = sum; |
| 1236 |
if (debug_level & 2) { |
| 1237 |
fmt::print(stderr, "\tProcessor {} has {} elements.\n", fmt::group_digits(i), |
| 1238 |
fmt::group_digits(sum)); |
| 1239 |
} |
| 1240 |
} |
| 1241 |
} |
| 1242 |
|
| 1243 |
template <typename INT> |
| 1244 |
void get_node_to_proc(Ioss::Region ®ion, std::vector<Ioss::Region *> &proc_region, |
| 1245 |
const std::vector<int> &elem_to_proc, std::vector<int> &node_to_proc, |
| 1246 |
std::vector<INT> &node_to_proc_pointer) |
| 1247 |
{ |
| 1248 |
progress(__func__); |
| 1249 |
// Process each element block connectivity to get the node_to_proc mapping. |
| 1250 |
|
| 1251 |
// The 'node_to_proc_pointer' vector maps the processor span in the |
| 1252 |
// node_to_proc vector. The processors that node 'node' (0-based) |
| 1253 |
// is on are: |
| 1254 |
// * begin = node_to_proc_pointer[node] |
| 1255 |
// * end = node_to_proc_pointer[node+1] |
| 1256 |
// * proc_list = node_to_proc[begin] .. node_to_proc[end-1] |
| 1257 |
// |
| 1258 |
|
| 1259 |
size_t proc_count = proc_region.size(); |
| 1260 |
|
| 1261 |
size_t node_count = region.get_property("node_count").get_int(); |
| 1262 |
if (node_count == 0) { |
| 1263 |
// Should never happen, but makes static analyzers happy... |
| 1264 |
return; |
| 1265 |
} |
| 1266 |
|
| 1267 |
std::vector<std::vector<int>> proc_node(node_count); |
| 1268 |
|
| 1269 |
// Assume that the majority of nodes will be on 2 or less |
| 1270 |
// processors (hopefully, most are on 1). |
| 1271 |
// Preallocate the proc_node[node] vector to 2 to minimize |
| 1272 |
// resizes... Use 'reserve' instead of 'resize' |
| 1273 |
for (size_t i = 0; i < node_count; i++) { |
| 1274 |
proc_node[i].reserve(2); |
| 1275 |
} |
| 1276 |
progress("\tProc_node reserved"); |
| 1277 |
|
| 1278 |
IOSS_MAYBE_UNUSED size_t sum_on_proc_count = 0; |
| 1279 |
Ioss::DatabaseIO *db = region.get_database(); |
| 1280 |
|
| 1281 |
const auto &ebs = region.get_element_blocks(); |
| 1282 |
size_t block_count = ebs.size(); |
| 1283 |
size_t offset = 0; |
| 1284 |
|
| 1285 |
std::vector<size_t> on_proc_count(proc_count); |
| 1286 |
for (size_t b = 0; b < block_count; b++) { |
| 1287 |
std::vector<INT> glob_conn; |
| 1288 |
size_t element_count = ebs[b]->entity_count(); |
| 1289 |
size_t element_nodes = ebs[b]->topology()->number_nodes(); |
| 1290 |
size_t block_id = ebs[b]->get_property("id").get_int(); |
| 1291 |
|
| 1292 |
// Do a 'partial_count' elements at a time... |
| 1293 |
if (element_count >= partial_count) { |
| 1294 |
int exoid = db->get_file_pointer(); |
| 1295 |
|
| 1296 |
glob_conn.resize(partial_count * element_nodes); |
| 1297 |
for (size_t beg = 1; beg <= element_count; beg += partial_count) { |
| 1298 |
size_t count = partial_count; |
| 1299 |
if (beg + count - 1 > element_count) { |
| 1300 |
count = element_count - beg + 1; |
| 1301 |
} |
| 1302 |
|
| 1303 |
ex_get_partial_conn(exoid, EX_ELEM_BLOCK, block_id, beg, count, Data(glob_conn), nullptr, |
| 1304 |
nullptr); |
| 1305 |
progress(fmt::format("\tpartial_conn-- start: {}\tcount: {}", fmt::group_digits(beg), |
| 1306 |
fmt::group_digits(count))); |
| 1307 |
populate_proc_node(count, offset, element_nodes, elem_to_proc, glob_conn, proc_node, |
| 1308 |
on_proc_count); |
| 1309 |
offset += count; |
| 1310 |
} |
| 1311 |
} |
| 1312 |
else { |
| 1313 |
ebs[b]->get_field_data("connectivity_raw", glob_conn); |
| 1314 |
populate_proc_node(element_count, offset, element_nodes, elem_to_proc, glob_conn, proc_node, |
| 1315 |
on_proc_count); |
| 1316 |
offset += element_count; |
| 1317 |
} |
| 1318 |
} |
| 1319 |
size_t spatial_dimension = region.get_property("spatial_dimension").get_int(); |
| 1320 |
for (size_t p = 0; p < proc_count; p++) { |
| 1321 |
auto *nb = new Ioss::NodeBlock(proc_region[p]->get_database(), "node_block1", |
| 1322 |
on_proc_count[p], spatial_dimension); |
| 1323 |
proc_region[p]->add(nb); |
| 1324 |
if (debug_level & 2) { |
| 1325 |
fmt::print(stderr, "\tProcessor {} has {} nodes.\n", fmt::group_digits(p), |
| 1326 |
fmt::group_digits(on_proc_count[p])); |
| 1327 |
} |
| 1328 |
sum_on_proc_count += on_proc_count[p]; |
| 1329 |
} |
| 1330 |
progress("\tProc_node populated"); |
| 1331 |
|
| 1332 |
// Have data for each node showing which processors it is on... |
| 1333 |
// proc_node[node].size() is number of processors for this node... |
| 1334 |
node_to_proc_pointer.reserve(node_count + 1); |
| 1335 |
|
| 1336 |
std::vector<size_t> proc_histo(17); |
| 1337 |
|
| 1338 |
size_t node_to_proc_pointer_size = 0; |
| 1339 |
for (size_t i = 0; i < node_count; i++) { |
| 1340 |
size_t num_procs = proc_node[i].size(); |
| 1341 |
if (num_procs == 0) { |
| 1342 |
fmt::print(stderr, "WARNING: Node {} is not connected to any elements.\n", |
| 1343 |
fmt::group_digits(i + 1)); |
| 1344 |
} |
| 1345 |
else if (num_procs < proc_histo.size()) { |
| 1346 |
proc_histo[num_procs]++; |
| 1347 |
} |
| 1348 |
else { |
| 1349 |
proc_histo[0]++; |
| 1350 |
} |
| 1351 |
|
| 1352 |
node_to_proc_pointer.push_back(node_to_proc_pointer_size); |
| 1353 |
node_to_proc_pointer_size += num_procs; |
| 1354 |
} |
| 1355 |
// Output histogram.. |
| 1356 |
fmt::print(stderr, "Processor count per node histogram:\n"); |
| 1357 |
for (size_t i = 1; i < proc_histo.size(); i++) { |
| 1358 |
if (proc_histo[i] > 0) { |
| 1359 |
fmt::print(stderr, "\tNodes on {:2} processors = {:12}\t({:2})%\n", fmt::group_digits(i), |
| 1360 |
fmt::group_digits(proc_histo[i]), |
| 1361 |
(proc_histo[i] * 100 + node_count / 2) / node_count); |
| 1362 |
} |
| 1363 |
} |
| 1364 |
if (proc_histo[0] > 0) { |
| 1365 |
fmt::print(stderr, "\tNodes on {} or more processors = {}\t({:2})%\n", |
| 1366 |
fmt::group_digits(proc_histo.size()), fmt::group_digits(proc_histo[0]), |
| 1367 |
(proc_histo[0] * 100 + node_count / 2) / node_count); |
| 1368 |
} |
| 1369 |
fmt::print(stderr, "\n"); |
| 1370 |
|
| 1371 |
node_to_proc_pointer.push_back(node_to_proc_pointer_size); |
| 1372 |
node_to_proc.reserve(node_to_proc_pointer_size); |
| 1373 |
progress("\tNode_to_proc reserved"); |
| 1374 |
assert(sum_on_proc_count == node_to_proc_pointer_size); |
| 1375 |
|
| 1376 |
for (auto &pn : proc_node) { |
| 1377 |
size_t num_procs = pn.size(); |
| 1378 |
for (size_t p = 0; p < num_procs; p++) { |
| 1379 |
node_to_proc.push_back(pn[p]); |
| 1380 |
} |
| 1381 |
} |
| 1382 |
assert(node_to_proc.size() == node_to_proc_pointer_size); |
| 1383 |
progress("\tNode_to_proc populated"); |
| 1384 |
} |
| 1385 |
|
| 1386 |
template <typename INT> |
| 1387 |
void slice(Ioss::Region ®ion, const std::string &nemfile, SystemInterface &interFace, |
| 1388 |
INT dummy) |
| 1389 |
{ |
| 1390 |
progress(__func__); |
| 1391 |
bool create_split_files = !interFace.outputDecompMap_ && !interFace.outputDecompField_; |
| 1392 |
|
| 1393 |
std::vector<Ioss::Region *> proc_region; |
| 1394 |
if (create_split_files) { |
| 1395 |
proc_region.resize(interFace.processor_count()); |
| 1396 |
} |
| 1397 |
bool ints64 = (sizeof(INT) == 8); |
| 1398 |
|
| 1399 |
Ioss::PropertyManager properties = set_properties(interFace); |
| 1400 |
|
| 1401 |
Ioss::chain_t<INT> element_chains; |
| 1402 |
std::vector<float> weights; |
| 1403 |
if (interFace.lineDecomp_) { |
| 1404 |
element_chains = |
| 1405 |
Ioss::generate_element_chains(region, interFace.lineSurfaceList_, debug_level, dummy); |
| 1406 |
progress("Ioss::generate_element_chains"); |
| 1407 |
|
| 1408 |
if (interFace.decomposition_method() == "rcb" || interFace.decomposition_method() == "rib" || |
| 1409 |
interFace.decomposition_method() == "hsfc") { |
| 1410 |
weights = Ioss::DecompUtils::line_decomp_weights( |
| 1411 |
element_chains, region.get_property("element_count").get_int()); |
| 1412 |
progress("generate_element_weights"); |
| 1413 |
} |
| 1414 |
} |
| 1415 |
|
| 1416 |
if (weights.empty()) { |
| 1417 |
weights.resize(region.get_property("element_count").get_int(), 1); |
| 1418 |
} |
| 1419 |
|
| 1420 |
double start = seacas_timer(); |
| 1421 |
auto elem_to_proc = decompose_elements(region, interFace, weights, dummy); |
| 1422 |
double end = seacas_timer(); |
| 1423 |
fmt::print(stderr, "Decompose elements = {:.5}\n", end - start); |
| 1424 |
progress("exit decompose_elements"); |
| 1425 |
|
| 1426 |
if (interFace.lineDecomp_) { |
| 1427 |
// Make sure all elements on a chain are on the same processor rank... |
| 1428 |
Ioss::DecompUtils::line_decomp_modify(element_chains, elem_to_proc, |
| 1429 |
interFace.processor_count()); |
| 1430 |
} |
| 1431 |
|
| 1432 |
if (debug_level & 32) { |
| 1433 |
auto work_per_rank = |
| 1434 |
Ioss::DecompUtils::get_work_per_rank(elem_to_proc, interFace.processor_count()); |
| 1435 |
auto avg_median = Ioss::DecompUtils::output_decomposition_statistics(work_per_rank); |
| 1436 |
Ioss::DecompUtils::output_histogram(work_per_rank, avg_median.first, avg_median.second); |
| 1437 |
} |
| 1438 |
|
| 1439 |
if (!create_split_files) { |
| 1440 |
Ioss::DatabaseIO *dbo = Ioss::IOFactory::create( |
| 1441 |
"exodus", nemfile, Ioss::WRITE_RESTART, Ioss::ParallelUtils::comm_world(), properties); |
| 1442 |
if (dbo == nullptr || !dbo->ok(true)) { |
| 1443 |
std::exit(EXIT_FAILURE); |
| 1444 |
} |
| 1445 |
if (ints64) { |
| 1446 |
dbo->set_int_byte_size_api(Ioss::USE_INT64_API); |
| 1447 |
} |
| 1448 |
|
| 1449 |
// NOTE: 'output_region' owns 'dbo' pointer at this time |
| 1450 |
Ioss::Region output_region(dbo, "region_2"); |
| 1451 |
|
| 1452 |
// Set the qa information... |
| 1453 |
output_region.property_add(Ioss::Property(std::string("code_name"), qainfo[0])); |
| 1454 |
output_region.property_add(Ioss::Property(std::string("code_version"), qainfo[2])); |
| 1455 |
|
| 1456 |
Ioss::MeshCopyOptions options{}; |
| 1457 |
options.ints_64_bit = sizeof(INT) == 64; |
| 1458 |
options.delete_timesteps = true; |
| 1459 |
options.data_storage_type = 1; |
| 1460 |
options.verbose = true; |
| 1461 |
|
| 1462 |
// Copy mesh portion of input region to the output region |
| 1463 |
Ioss::copy_database(region, output_region, options); |
| 1464 |
|
| 1465 |
// KLUGE: The metadata has already been written on |
| 1466 |
// output_region, but we couldn't define the maps until now, so |
| 1467 |
// need to update the metadata with map information and hope |
| 1468 |
// that no other maps exist on the database... |
| 1469 |
if (interFace.outputDecompMap_) { |
| 1470 |
bool line_decomp = interFace.lineDecomp_; |
| 1471 |
add_decomp_map(output_region, interFace.decomposition_variable(), line_decomp); |
| 1472 |
output_decomp_map(output_region, elem_to_proc, element_chains, |
| 1473 |
interFace.decomposition_variable(), line_decomp); |
| 1474 |
progress("output_decomp_map"); |
| 1475 |
} |
| 1476 |
|
| 1477 |
if (interFace.outputDecompField_) { |
| 1478 |
bool line_decomp = interFace.lineDecomp_; |
| 1479 |
add_decomp_field(output_region, interFace.decomposition_variable(), line_decomp); |
| 1480 |
output_decomp_field(output_region, elem_to_proc, element_chains, |
| 1481 |
interFace.decomposition_variable(), line_decomp); |
| 1482 |
progress("output_decomp_field"); |
| 1483 |
} |
| 1484 |
|
| 1485 |
return; |
| 1486 |
} |
| 1487 |
|
| 1488 |
bool close_files = interFace.processor_count() + 1 > interFace.max_files(); |
| 1489 |
for (size_t i = 0; i < interFace.processor_count(); i++) { |
| 1490 |
std::string outfile = Ioss::Utils::decode_filename(nemfile, i, interFace.processor_count()); |
| 1491 |
Ioss::DatabaseIO *dbo = Ioss::IOFactory::create( |
| 1492 |
"exodus", outfile, Ioss::WRITE_RESTART, Ioss::ParallelUtils::comm_world(), properties); |
| 1493 |
if (ints64) { |
| 1494 |
dbo->set_int_byte_size_api(Ioss::USE_INT64_API); |
| 1495 |
} |
| 1496 |
|
| 1497 |
proc_region[i] = new Ioss::Region(dbo); |
| 1498 |
proc_region[i]->begin_mode(Ioss::STATE_DEFINE_MODEL); |
| 1499 |
if (close_files) { |
| 1500 |
proc_region[i]->get_database()->closeDatabase(); |
| 1501 |
} |
| 1502 |
} |
| 1503 |
|
| 1504 |
start = seacas_timer(); |
| 1505 |
// Build the proc_elem_block_cnt[i][j] vector. |
| 1506 |
// Gives number of elements in block i on processor j |
| 1507 |
size_t block_count = region.get_property("element_block_count").get_int(); |
| 1508 |
std::vector<std::vector<INT>> proc_elem_block_cnt(block_count + 1); |
| 1509 |
for (auto &pebc : proc_elem_block_cnt) { |
| 1510 |
pebc.resize(interFace.processor_count()); |
| 1511 |
} |
| 1512 |
get_proc_elem_block_count(region, elem_to_proc, proc_elem_block_cnt); |
| 1513 |
end = seacas_timer(); |
| 1514 |
|
| 1515 |
fmt::print(stderr, "Calculate elements per element block on each processor = {:.5}\n", |
| 1516 |
end - start); |
| 1517 |
|
| 1518 |
// Create element blocks for each processor... |
| 1519 |
for (size_t p = 0; p < interFace.processor_count(); p++) { |
| 1520 |
const auto &ebs = region.get_element_blocks(); |
| 1521 |
size_t bc = ebs.size(); |
| 1522 |
for (size_t b = 0; b < bc; b++) { |
| 1523 |
std::string type = ebs[b]->topology()->name(); |
| 1524 |
auto *eb = new Ioss::ElementBlock(proc_region[p]->get_database(), ebs[b]->name(), type, |
| 1525 |
proc_elem_block_cnt[b][p]); |
| 1526 |
proc_region[p]->add(eb); |
| 1527 |
} |
| 1528 |
} |
| 1529 |
|
| 1530 |
// Now that we have the elements on each processor and the element |
| 1531 |
// blocks those elements are in, can generate the node to proc list... |
| 1532 |
start = seacas_timer(); |
| 1533 |
std::vector<int> node_to_proc; |
| 1534 |
std::vector<INT> node_to_proc_pointer; |
| 1535 |
get_node_to_proc(region, proc_region, elem_to_proc, node_to_proc, node_to_proc_pointer); |
| 1536 |
end = seacas_timer(); |
| 1537 |
fmt::print(stderr, "Node Categorization Time = {:.5}\n", end - start); |
| 1538 |
|
| 1539 |
// Communication map data -- interior/border nodes |
| 1540 |
start = seacas_timer(); |
| 1541 |
define_communication_data(region, proc_region, node_to_proc, node_to_proc_pointer); |
| 1542 |
end = seacas_timer(); |
| 1543 |
fmt::print(stderr, "Communication Data Definitions = {:.5}\n", end - start); |
| 1544 |
|
| 1545 |
// Determine nodeset distribution to processor regions. |
| 1546 |
start = seacas_timer(); |
| 1547 |
get_nodesets(region, proc_region, node_to_proc, node_to_proc_pointer); |
| 1548 |
end = seacas_timer(); |
| 1549 |
fmt::print(stderr, "Get nodeset data = {:.5}\n", end - start); |
| 1550 |
|
| 1551 |
start = seacas_timer(); |
| 1552 |
get_sidesets(region, proc_region, elem_to_proc, (INT)0); |
| 1553 |
end = seacas_timer(); |
| 1554 |
fmt::print(stderr, "Get sideset data = {:.5}\n", end - start); |
| 1555 |
|
| 1556 |
start = seacas_timer(); |
| 1557 |
double start_comb = start; |
| 1558 |
fmt::print(stderr, "Begin writing output files\n"); |
| 1559 |
size_t proc_count = interFace.processor_count(); |
| 1560 |
|
| 1561 |
// Output in processor chunks of size <= max_files so can keep all files open.... |
| 1562 |
size_t max_files = interFace.max_files(); |
| 1563 |
size_t chunks = (proc_count + max_files - 1) / max_files; |
| 1564 |
size_t size_per_chunk = (proc_count + chunks - 1) / chunks; |
| 1565 |
if (chunks > 1) { |
| 1566 |
fmt::print(stderr, |
| 1567 |
"\nMax open files = {}; processing files in {} chunks of size {} to maximize " |
| 1568 |
"performance.\n", |
| 1569 |
max_files, chunks, size_per_chunk); |
| 1570 |
} |
| 1571 |
for (size_t chunk = 0; chunk < chunks; chunk++) { |
| 1572 |
size_t proc_begin = chunk * size_per_chunk; |
| 1573 |
size_t proc_size = size_per_chunk; |
| 1574 |
if (proc_begin + proc_size > proc_count) { |
| 1575 |
proc_size = proc_count - proc_begin; |
| 1576 |
} |
| 1577 |
fmt::print(stderr, "\nProcessor range {} to {}\n", fmt::group_digits(proc_begin), |
| 1578 |
fmt::group_digits(proc_begin + proc_size - 1)); |
| 1579 |
|
| 1580 |
for (size_t p = proc_begin; p < proc_begin + proc_size; p++) { |
| 1581 |
Ioss::transfer_coordinate_frames(region, *proc_region[p]); |
| 1582 |
Ioss::transfer_assemblies(region, *proc_region[p], Ioss::MeshCopyOptions{}, 0); |
| 1583 |
proc_region[p]->synchronize_id_and_name(®ion); |
| 1584 |
proc_region[p]->end_mode(Ioss::STATE_DEFINE_MODEL); |
| 1585 |
if (interFace.lineDecomp_) { |
| 1586 |
add_chain_maps(*proc_region[p]); |
| 1587 |
} |
| 1588 |
proc_region[p]->begin_mode(Ioss::STATE_MODEL); |
| 1589 |
proc_progress(p, proc_count); |
| 1590 |
} |
| 1591 |
end = seacas_timer(); |
| 1592 |
fmt::print(stderr, "\tDefine output databases = {:.5}\n", end - start); |
| 1593 |
|
| 1594 |
// Generate and output node map... |
| 1595 |
#if 1 |
| 1596 |
start = seacas_timer(); |
| 1597 |
output_node_map(region, proc_region, node_to_proc, node_to_proc_pointer, proc_begin, |
| 1598 |
proc_size); |
| 1599 |
end = seacas_timer(); |
| 1600 |
fmt::print(stderr, "\tNode Map Output = {:.5}\n", end - start); |
| 1601 |
#else |
| 1602 |
start = seacas_timer(); |
| 1603 |
output_global_node_map(region, proc_region, node_to_proc, node_to_proc_pointer, proc_begin, |
| 1604 |
proc_size); |
| 1605 |
end = seacas_timer(); |
| 1606 |
fmt::print(stderr, "\tGlobal Node Map Output = {:.5}\n", end - start); |
| 1607 |
#endif |
| 1608 |
|
| 1609 |
start = seacas_timer(); |
| 1610 |
output_element_map(region, proc_region, elem_to_proc, proc_begin, proc_size, (INT)1); |
| 1611 |
end = seacas_timer(); |
| 1612 |
fmt::print(stderr, "\tElement Map Output = {:.5}\n", end - start); |
| 1613 |
|
| 1614 |
start = seacas_timer(); |
| 1615 |
output_communication_map(region, proc_region, node_to_proc, node_to_proc_pointer, proc_begin, |
| 1616 |
proc_size); |
| 1617 |
end = seacas_timer(); |
| 1618 |
fmt::print(stderr, "\tCommunication map Output = {:.5}\n", end - start); |
| 1619 |
|
| 1620 |
start = seacas_timer(); |
| 1621 |
output_connectivity(region, proc_region, elem_to_proc, proc_begin, proc_size, (INT)1); |
| 1622 |
end = seacas_timer(); |
| 1623 |
fmt::print(stderr, "\tConnectivity Output = {:.5}\n", end - start); |
| 1624 |
|
| 1625 |
start = seacas_timer(); |
| 1626 |
#if 0 |
| 1627 |
output_coordinates(region, proc_region, node_to_proc, node_to_proc_pointer, proc_begin, |
| 1628 |
proc_size); |
| 1629 |
#else |
| 1630 |
output_coordinates_c(region, proc_region, node_to_proc, node_to_proc_pointer, proc_begin, |
| 1631 |
proc_size); |
| 1632 |
#endif |
| 1633 |
end = seacas_timer(); |
| 1634 |
fmt::print(stderr, "\tCoordinates Output = {:.5}\n", end - start); |
| 1635 |
|
| 1636 |
start = seacas_timer(); |
| 1637 |
output_nodesets(region, proc_region, node_to_proc, node_to_proc_pointer, proc_begin, |
| 1638 |
proc_size); |
| 1639 |
end = seacas_timer(); |
| 1640 |
fmt::print(stderr, "\tNodeset Output = {:.5}\n", end - start); |
| 1641 |
|
| 1642 |
start = seacas_timer(); |
| 1643 |
output_sidesets(region, proc_region, elem_to_proc, proc_begin, proc_size, (INT)0); |
| 1644 |
end = seacas_timer(); |
| 1645 |
fmt::print(stderr, "\tSideset Output = {:.5}\n", end - start); |
| 1646 |
|
| 1647 |
if (interFace.lineDecomp_) { |
| 1648 |
start = seacas_timer(); |
| 1649 |
output_chain_maps(proc_region, element_chains, elem_to_proc, proc_begin, proc_size, (INT)0); |
| 1650 |
end = seacas_timer(); |
| 1651 |
fmt::print(stderr, "\tChain Map Output = {:.5}\n", end - start); |
| 1652 |
} |
| 1653 |
|
| 1654 |
// Close all files... |
| 1655 |
start = seacas_timer(); |
| 1656 |
for (size_t p = proc_begin; p < proc_begin + proc_size; p++) { |
| 1657 |
proc_region[p]->end_mode(Ioss::STATE_MODEL); |
| 1658 |
delete proc_region[p]; |
| 1659 |
} |
| 1660 |
end = seacas_timer(); |
| 1661 |
fmt::print(stderr, "\tClose and finalize processor {} to {} output databases = {:.5}\n", |
| 1662 |
proc_begin, proc_begin + proc_size - 1, end - start); |
| 1663 |
} |
| 1664 |
end = seacas_timer(); |
| 1665 |
fmt::print(stderr, "\nTotal time to write output files = {:.5} ({:.5} per file)\n", |
| 1666 |
end - start_comb, (end - start_comb) / interFace.processor_count()); |
| 1667 |
} |
| 1668 |
|
| 1669 |
void filename_substitution(std::string &filename, const SystemInterface &interFace) |
| 1670 |
{ |
| 1671 |
// See if filename contains "%P" which is replaced by the number of processors... |
| 1672 |
// Assumes that %P only occurs once... |
| 1673 |
// filename is changed. |
| 1674 |
size_t pos = filename.find("%P"); |
| 1675 |
if (pos != std::string::npos) { |
| 1676 |
// Found the characters... Replace with the processor count... |
| 1677 |
size_t num_proc = interFace.processor_count(); |
| 1678 |
std::string tmp(filename, 0, pos); |
| 1679 |
tmp += std::to_string(num_proc); |
| 1680 |
tmp += filename.substr(pos + 2); |
| 1681 |
filename = tmp; |
| 1682 |
} |
| 1683 |
|
| 1684 |
// If contains %M, replace with the decomposition method. |
| 1685 |
pos = filename.find("%M"); |
| 1686 |
if (pos != std::string::npos) { |
| 1687 |
// Found the characters... Replace with the input file basename... |
| 1688 |
const std::string &method_name = interFace.decomposition_method(); |
| 1689 |
std::string tmp(filename, 0, pos); |
| 1690 |
tmp += method_name; |
| 1691 |
tmp += filename.substr(pos + 2); |
| 1692 |
filename = tmp; |
| 1693 |
} |
| 1694 |
} |
| 1695 |
|
| 1696 |
} // namespace |