| 1 |
/* |
| 2 |
* Copyright(C) 1999-2021, 2023, 2024 National Technology & Engineering Solutions |
| 3 |
* of Sandia, LLC (NTESS). Under the terms of Contract DE-NA0003525 with |
| 4 |
* NTESS, the U.S. Government retains certain rights in this software. |
| 5 |
* |
| 6 |
* See packages/seacas/LICENSE for details |
| 7 |
*/ |
| 8 |
|
| 9 |
#include "elb.h" // for Machine_Description, etc |
| 10 |
#include "elb_allo.h" // for array_alloc |
| 11 |
#include "elb_elem.h" // for NNODES, get_elem_info |
| 12 |
#include "elb_err.h" // for Gen_Error, error_lev |
| 13 |
#include "elb_output.h" |
| 14 |
#include "elb_util.h" // for gds_qsort, qsort2, in_list, etc |
| 15 |
#include "fmt/chrono.h" |
| 16 |
#include "fmt/ostream.h" |
| 17 |
#include "scopeguard.h" |
| 18 |
#include "vector_data.h" |
| 19 |
#include <copy_string_cpp.h> |
| 20 |
#include <cstddef> // for size_t, nullptr |
| 21 |
#include <cstdlib> // for free, malloc, realloc |
| 22 |
#include <cstring> // for strcat, strlen, etc |
| 23 |
#include <ctime> // for asctime, localtime, time, etc |
| 24 |
#include <exodusII.h> // for ex_close, ex_opts, etc |
| 25 |
#include <iostream> |
| 26 |
#include <vector> // for vector |
| 27 |
|
| 28 |
namespace { |
| 29 |
std::string remove_extension(const std::string &filename) |
| 30 |
{ |
| 31 |
// Strip off the extension |
| 32 |
size_t ind = filename.find_last_of('.', filename.size()); |
| 33 |
if (ind != std::string::npos) { |
| 34 |
return filename.substr(0, ind); |
| 35 |
} |
| 36 |
return filename; |
| 37 |
} |
| 38 |
} // namespace |
| 39 |
|
| 40 |
/*****************************************************************************/ |
| 41 |
/*****************************************************************************/ |
| 42 |
/*****************************************************************************/ |
| 43 |
/* This function outputs a load balance file using the ExodusII and NemesisI |
| 44 |
* API. |
| 45 |
*****************************************************************************/ |
| 46 |
template int write_nemesis(std::string &nemI_out_file, Machine_Description *machine, |
| 47 |
Problem_Description *problem, Mesh_Description<int> *mesh, |
| 48 |
LB_Description<int> *lb, Sphere_Info *sphere); |
| 49 |
template int write_nemesis(std::string &nemI_out_file, Machine_Description *machine, |
| 50 |
Problem_Description *problem, Mesh_Description<int64_t> *mesh, |
| 51 |
LB_Description<int64_t> *lb, Sphere_Info *sphere); |
| 52 |
|
| 53 |
template <typename INT> |
| 54 |
int write_nemesis(std::string &nemI_out_file, Machine_Description *machine, |
| 55 |
Problem_Description *problem, Mesh_Description<INT> *mesh, |
| 56 |
LB_Description<INT> *lb, Sphere_Info *sphere) |
| 57 |
{ |
| 58 |
int exoid; |
| 59 |
std::string method1{}, method2{}; |
| 60 |
std::string title; |
| 61 |
|
| 62 |
int cpu_ws = sizeof(float); |
| 63 |
int io_ws = sizeof(float); |
| 64 |
|
| 65 |
fmt::print("Outputting load balance to file {}\n", nemI_out_file); |
| 66 |
|
| 67 |
/* Create the load balance file */ |
| 68 |
/* Attempt to create a netcdf4-format file; if it fails, then assume |
| 69 |
that the netcdf library does not support that mode and fall back |
| 70 |
to classic netcdf3 format. If that fails, issue an error and |
| 71 |
return failure. |
| 72 |
*/ |
| 73 |
int mode3 = EX_CLOBBER; |
| 74 |
int mode4 = mode3 | EX_NETCDF4 | EX_NOCLASSIC | problem->int64db | problem->int64api; |
| 75 |
|
| 76 |
ex_opts(EX_DEFAULT); // Eliminate misleading error if the first ex_create fails, but the second |
| 77 |
// succeeds. |
| 78 |
if ((exoid = ex_create(nemI_out_file.c_str(), mode4, &cpu_ws, &io_ws)) < 0) { |
| 79 |
/* If int64api or int64db non-zero, then netcdf-4 format is required, so |
| 80 |
fail now... |
| 81 |
*/ |
| 82 |
if (problem->int64db | problem->int64api) { |
| 83 |
Gen_Error(0, "fatal: failed to create Nemesis netcdf-4 file"); |
| 84 |
return 0; |
| 85 |
} |
| 86 |
if ((exoid = ex_create(nemI_out_file.c_str(), mode3, &cpu_ws, &io_ws)) < 0) { |
| 87 |
Gen_Error(0, "fatal: failed to create Nemesis file"); |
| 88 |
return 0; |
| 89 |
} |
| 90 |
} |
| 91 |
ON_BLOCK_EXIT(ex_close, exoid); |
| 92 |
|
| 93 |
/* Set the error reporting value */ |
| 94 |
if (error_lev > 1) { |
| 95 |
ex_opts(EX_VERBOSE | EX_DEBUG); |
| 96 |
} |
| 97 |
else { |
| 98 |
ex_opts(EX_VERBOSE); |
| 99 |
} |
| 100 |
|
| 101 |
/* Enable compression (if netcdf-4) */ |
| 102 |
ex_set_option(exoid, EX_OPT_COMPRESSION_LEVEL, 1); |
| 103 |
ex_set_option(exoid, EX_OPT_COMPRESSION_SHUFFLE, 1); |
| 104 |
|
| 105 |
/* Create the title */ |
| 106 |
if (problem->type == DecompType::NODAL) { |
| 107 |
method1 = "nodal"; |
| 108 |
} |
| 109 |
else { |
| 110 |
method1 = "elemental"; |
| 111 |
} |
| 112 |
|
| 113 |
title = fmt::format("nem_slice {} load balance file", method1); |
| 114 |
|
| 115 |
method1 = "method1: "; |
| 116 |
method2 = "method2: "; |
| 117 |
|
| 118 |
switch (lb->type) { |
| 119 |
case Balance::MULTIKL: |
| 120 |
method1 += "Multilevel-KL decomposition"; |
| 121 |
method2 += "With Kernighan-Lin refinement"; |
| 122 |
break; |
| 123 |
case Balance::SPECTRAL: method1 += "Spectral decomposition"; break; |
| 124 |
case Balance::INERTIAL: method1 += "Inertial decomposition"; break; |
| 125 |
case Balance::ZPINCH: method1 += "ZPINCH decomposition"; break; |
| 126 |
case Balance::BRICK: method1 += "BRICK decomposition"; break; |
| 127 |
case Balance::ZOLTAN_RCB: method1 += "RCB decomposition"; break; |
| 128 |
case Balance::ZOLTAN_RIB: method1 += "RIB decomposition"; break; |
| 129 |
case Balance::ZOLTAN_HSFC: method1 += "HSFC decomposition"; break; |
| 130 |
case Balance::LINEAR: method1 += "Linear decomposition"; break; |
| 131 |
case Balance::RANDOM: method1 += "Random decomposition"; break; |
| 132 |
case Balance::SCATTERED: method1 += "Scattered decomposition"; break; |
| 133 |
case Balance::INFILE: method1 += "Infile decomposition"; break; |
| 134 |
case Balance::INVALID: method1 += "ERROR: Invalid decomposition"; break; |
| 135 |
default:; // do nothing |
| 136 |
} |
| 137 |
|
| 138 |
if (lb->refine == Balance::KL_REFINE && lb->type != Balance::MULTIKL) { |
| 139 |
method2 += "with Kernighan-Lin refinement"; |
| 140 |
} |
| 141 |
else if (lb->type != Balance::MULTIKL) { |
| 142 |
method2 += "no refinement"; |
| 143 |
} |
| 144 |
|
| 145 |
switch (lb->num_sects) { |
| 146 |
case 1: method1 += " via bisection"; break; |
| 147 |
case 2: method1 += " via quadrasection"; break; |
| 148 |
case 3: method1 += " via octasection"; break; |
| 149 |
} |
| 150 |
|
| 151 |
/* Do some sorting */ |
| 152 |
for (int proc = 0; proc < machine->num_procs; proc++) { |
| 153 |
|
| 154 |
/* Sort node maps */ |
| 155 |
gds_qsort(Data(lb->int_nodes[proc]), lb->int_nodes[proc].size()); |
| 156 |
if (problem->type == DecompType::NODAL) { |
| 157 |
sort2(lb->ext_nodes[proc].size(), Data(lb->ext_nodes[proc]), Data(lb->ext_procs[proc])); |
| 158 |
} |
| 159 |
|
| 160 |
/* Sort element maps */ |
| 161 |
gds_qsort(Data(lb->int_elems[proc]), lb->int_elems[proc].size()); |
| 162 |
} |
| 163 |
|
| 164 |
/* Output the info records */ |
| 165 |
char *info[3]; |
| 166 |
info[0] = const_cast<char *>(title.c_str()); |
| 167 |
info[1] = const_cast<char *>(method1.c_str()); |
| 168 |
info[2] = const_cast<char *>(method2.c_str()); |
| 169 |
|
| 170 |
if (ex_put_info(exoid, 3, info) < 0) { |
| 171 |
Gen_Error(0, "warning: output of info records failed"); |
| 172 |
} |
| 173 |
|
| 174 |
/* Generate a QA record for the utility */ |
| 175 |
time_t time_val = time(nullptr); |
| 176 |
auto *lt = std::localtime(&time_val); |
| 177 |
std::string time = fmt::format("{:%H:%M:%S}", *lt); |
| 178 |
std::string date = fmt::format("{:%Y/%m/%d}", *lt); |
| 179 |
|
| 180 |
char qa_date[32]; |
| 181 |
char qa_time[32]; |
| 182 |
char qa_name[MAX_STR_LENGTH]; |
| 183 |
char qa_vers[32]; |
| 184 |
|
| 185 |
copy_string(qa_time, time); |
| 186 |
copy_string(qa_date, date); |
| 187 |
copy_string(qa_name, UTIL_NAME); |
| 188 |
copy_string(qa_vers, ELB_VERSION); |
| 189 |
|
| 190 |
char **lqa_record = reinterpret_cast<char **>(array_alloc(1, 4, sizeof(char *))); |
| 191 |
for (int i2 = 0; i2 < 4; i2++) { |
| 192 |
lqa_record[i2] = reinterpret_cast<char *>(array_alloc(1, MAX_STR_LENGTH + 1, sizeof(char))); |
| 193 |
} |
| 194 |
|
| 195 |
copy_string(lqa_record[0], qa_name, MAX_STR_LENGTH + 1); |
| 196 |
copy_string(lqa_record[1], qa_vers, MAX_STR_LENGTH + 1); |
| 197 |
copy_string(lqa_record[2], qa_date, MAX_STR_LENGTH + 1); |
| 198 |
copy_string(lqa_record[3], qa_time, MAX_STR_LENGTH + 1); |
| 199 |
|
| 200 |
fmt::print("QA Record:\n"); |
| 201 |
for (int i2 = 0; i2 < 4; i2++) { |
| 202 |
fmt::print("\t{}\n", lqa_record[i2]); |
| 203 |
} |
| 204 |
|
| 205 |
if (ex_put_qa(exoid, 1, reinterpret_cast<char *(*)[4]>(&lqa_record[0])) < 0) { |
| 206 |
Gen_Error(0, "fatal: unable to output QA records"); |
| 207 |
return 0; |
| 208 |
} |
| 209 |
|
| 210 |
/* free up memory */ |
| 211 |
for (int i2 = 0; i2 < 4; i2++) { |
| 212 |
free(lqa_record[i2]); |
| 213 |
} |
| 214 |
|
| 215 |
free(lqa_record); |
| 216 |
|
| 217 |
/* Output the the initial Nemesis global information */ |
| 218 |
if (ex_put_init_global(exoid, mesh->num_nodes, mesh->num_elems, mesh->num_el_blks, 0, 0) < 0) { |
| 219 |
Gen_Error(0, "fatal: failed to output initial Nemesis parameters"); |
| 220 |
return 0; |
| 221 |
} |
| 222 |
|
| 223 |
ex_put_eb_info_global(exoid, Data(mesh->eb_ids), Data(mesh->eb_cnts)); |
| 224 |
|
| 225 |
/* Set up dummy arrays for output */ |
| 226 |
std::vector<INT> num_nmap_cnts(machine->num_procs); |
| 227 |
std::vector<INT> num_emap_cnts(machine->num_procs); |
| 228 |
|
| 229 |
if (problem->type == DecompType::NODAL) { |
| 230 |
/* need to check and make sure that there really are comm maps */ |
| 231 |
for (int cnt = 0; cnt < machine->num_procs; cnt++) { |
| 232 |
if (!lb->bor_nodes[cnt].empty()) { |
| 233 |
num_nmap_cnts[cnt] = 1; |
| 234 |
} |
| 235 |
} |
| 236 |
} |
| 237 |
else { /* Elemental load balance */ |
| 238 |
if (problem->num_vertices > sphere->num) { |
| 239 |
/* need to check and make sure that there really are comm maps */ |
| 240 |
for (int cnt = 0; cnt < machine->num_procs; cnt++) { |
| 241 |
if (!lb->bor_nodes[cnt].empty()) { |
| 242 |
num_nmap_cnts[cnt] = 1; |
| 243 |
} |
| 244 |
} |
| 245 |
for (int cnt = 0; cnt < machine->num_procs; cnt++) { |
| 246 |
if (!lb->bor_elems[cnt].empty()) { |
| 247 |
num_emap_cnts[cnt] = 1; |
| 248 |
} |
| 249 |
} |
| 250 |
} |
| 251 |
} |
| 252 |
|
| 253 |
if (ex_put_init_info(exoid, machine->num_procs, machine->num_procs, (char *)"s") < 0) { |
| 254 |
Gen_Error(0, "fatal: unable to output init info"); |
| 255 |
return 0; |
| 256 |
} |
| 257 |
|
| 258 |
// Need to create 5 arrays with the sizes of lb->int_nodes[i].size()... |
| 259 |
{ |
| 260 |
std::vector<INT> ins(machine->num_procs); |
| 261 |
std::vector<INT> bns(machine->num_procs); |
| 262 |
std::vector<INT> ens(machine->num_procs); |
| 263 |
std::vector<INT> ies(machine->num_procs); |
| 264 |
std::vector<INT> bes(machine->num_procs); |
| 265 |
|
| 266 |
for (int iproc = 0; iproc < machine->num_procs; iproc++) { |
| 267 |
ins[iproc] = lb->int_nodes[iproc].size(); |
| 268 |
bns[iproc] = lb->bor_nodes[iproc].size(); |
| 269 |
ens[iproc] = lb->ext_nodes[iproc].size(); |
| 270 |
ies[iproc] = lb->int_elems[iproc].size(); |
| 271 |
bes[iproc] = lb->bor_elems[iproc].size(); |
| 272 |
} |
| 273 |
|
| 274 |
if (ex_put_loadbal_param_cc(exoid, Data(ins), Data(bns), Data(ens), Data(ies), Data(bes), |
| 275 |
Data(num_nmap_cnts), Data(num_emap_cnts)) < 0) { |
| 276 |
Gen_Error(0, "fatal: unable to output load-balance parameters"); |
| 277 |
return 0; |
| 278 |
} |
| 279 |
} |
| 280 |
|
| 281 |
if (problem->type == DecompType::NODAL) /* Nodal load balance output */ |
| 282 |
{ |
| 283 |
/* Set up for the concatenated communication map parameters */ |
| 284 |
std::vector<INT> node_proc_ptr(machine->num_procs + 1); |
| 285 |
std::vector<INT> node_cmap_ids_cc(machine->num_procs); |
| 286 |
std::vector<INT> node_cmap_cnts_cc(machine->num_procs); |
| 287 |
|
| 288 |
node_proc_ptr[0] = 0; |
| 289 |
for (int proc = 0; proc < machine->num_procs; proc++) { |
| 290 |
node_proc_ptr[proc + 1] = node_proc_ptr[proc] + 1; |
| 291 |
node_cmap_cnts_cc[proc] = lb->ext_nodes[proc].size(); |
| 292 |
node_cmap_ids_cc[proc] = 1; |
| 293 |
} |
| 294 |
|
| 295 |
/* Output the communication map parameters */ |
| 296 |
if (ex_put_cmap_params_cc(exoid, Data(node_cmap_ids_cc), Data(node_cmap_cnts_cc), |
| 297 |
Data(node_proc_ptr), nullptr, nullptr, nullptr) < 0) { |
| 298 |
Gen_Error(0, "fatal: unable to output communication map parameters"); |
| 299 |
return 0; |
| 300 |
} |
| 301 |
|
| 302 |
/* Output the node and element maps */ |
| 303 |
for (int proc = 0; proc < machine->num_procs; proc++) { |
| 304 |
/* Output the nodal map */ |
| 305 |
if (ex_put_processor_node_maps(exoid, Data(lb->int_nodes[proc]), Data(lb->bor_nodes[proc]), |
| 306 |
Data(lb->ext_nodes[proc]), proc) < 0) { |
| 307 |
Gen_Error(0, "fatal: failed to output node map"); |
| 308 |
return 0; |
| 309 |
} |
| 310 |
|
| 311 |
/* Output the elemental map */ |
| 312 |
if (ex_put_processor_elem_maps(exoid, Data(lb->int_elems[proc]), nullptr, proc) < 0) { |
| 313 |
Gen_Error(0, "fatal: failed to output element map"); |
| 314 |
return 0; |
| 315 |
} |
| 316 |
|
| 317 |
/* |
| 318 |
* Reorder the nodal communication maps so that they are ordered |
| 319 |
* by processor and then by global ID. |
| 320 |
*/ |
| 321 |
|
| 322 |
/* This is a 2-key sort */ |
| 323 |
qsort2(Data(lb->ext_procs[proc]), Data(lb->ext_nodes[proc]), lb->ext_nodes[proc].size()); |
| 324 |
|
| 325 |
/* Output the nodal communication map */ |
| 326 |
if (ex_put_node_cmap(exoid, 1, Data(lb->ext_nodes[proc]), Data(lb->ext_procs[proc]), proc) < |
| 327 |
0) { |
| 328 |
Gen_Error(0, "fatal: failed to output nodal communication map"); |
| 329 |
return 0; |
| 330 |
} |
| 331 |
|
| 332 |
} /* End "for(proc=0; proc < machine->num_procs; proc++)" */ |
| 333 |
} |
| 334 |
else if (problem->type == DecompType::ELEMENTAL) /* Elemental load balance output */ |
| 335 |
{ |
| 336 |
std::vector<INT> node_proc_ptr(machine->num_procs + 1); |
| 337 |
std::vector<INT> node_cmap_ids_cc(machine->num_procs); |
| 338 |
std::vector<INT> node_cmap_cnts_cc(machine->num_procs); |
| 339 |
|
| 340 |
node_proc_ptr[0] = 0; |
| 341 |
for (int proc = 0; proc < machine->num_procs; proc++) { |
| 342 |
node_proc_ptr[proc + 1] = node_proc_ptr[proc] + 1; |
| 343 |
|
| 344 |
node_cmap_cnts_cc[proc] = 0; |
| 345 |
for (size_t cnt = 0; cnt < lb->bor_nodes[proc].size(); cnt++) { |
| 346 |
node_cmap_cnts_cc[proc] += lb->born_procs[proc][cnt].size(); |
| 347 |
} |
| 348 |
|
| 349 |
node_cmap_ids_cc[proc] = 1; |
| 350 |
} |
| 351 |
|
| 352 |
std::vector<INT> elem_proc_ptr(machine->num_procs + 1); |
| 353 |
std::vector<INT> elem_cmap_ids_cc(machine->num_procs); |
| 354 |
std::vector<INT> elem_cmap_cnts_cc(machine->num_procs); |
| 355 |
|
| 356 |
elem_proc_ptr[0] = 0; |
| 357 |
for (int proc = 0; proc < machine->num_procs; proc++) { |
| 358 |
elem_proc_ptr[proc + 1] = elem_proc_ptr[proc] + 1; |
| 359 |
elem_cmap_cnts_cc[proc] = lb->e_cmap_elems[proc].size(); |
| 360 |
elem_cmap_ids_cc[proc] = 1; |
| 361 |
} |
| 362 |
|
| 363 |
/* Output the communication map parameters */ |
| 364 |
if (ex_put_cmap_params_cc(exoid, Data(node_cmap_ids_cc), Data(node_cmap_cnts_cc), |
| 365 |
Data(node_proc_ptr), Data(elem_cmap_ids_cc), Data(elem_cmap_cnts_cc), |
| 366 |
Data(elem_proc_ptr)) < 0) { |
| 367 |
Gen_Error(0, "fatal: unable to output communication map parameters"); |
| 368 |
return 0; |
| 369 |
} |
| 370 |
|
| 371 |
/* Output the node and element maps */ |
| 372 |
for (int proc = 0; proc < machine->num_procs; proc++) { |
| 373 |
/* Output the nodal map */ |
| 374 |
if (ex_put_processor_node_maps(exoid, Data(lb->int_nodes[proc]), Data(lb->bor_nodes[proc]), |
| 375 |
nullptr, proc) < 0) { |
| 376 |
Gen_Error(0, "fatal: failed to output node map"); |
| 377 |
return 0; |
| 378 |
} |
| 379 |
|
| 380 |
/* Output the elemental map */ |
| 381 |
if (ex_put_processor_elem_maps(exoid, Data(lb->int_elems[proc]), Data(lb->bor_elems[proc]), |
| 382 |
proc) < 0) { |
| 383 |
Gen_Error(0, "fatal: failed to output element map"); |
| 384 |
return 0; |
| 385 |
} |
| 386 |
|
| 387 |
/* |
| 388 |
* Build a nodal communication map from the list of border nodes |
| 389 |
* and their associated processors and side IDs. |
| 390 |
*/ |
| 391 |
size_t nsize = 0; |
| 392 |
for (size_t cnt = 0; cnt < lb->bor_nodes[proc].size(); cnt++) { |
| 393 |
nsize += lb->born_procs[proc][cnt].size(); |
| 394 |
} |
| 395 |
|
| 396 |
if (nsize > 0) { |
| 397 |
std::vector<INT> n_cmap_nodes(nsize); |
| 398 |
std::vector<INT> n_cmap_procs(nsize); |
| 399 |
|
| 400 |
size_t cnt3 = 0; |
| 401 |
for (size_t cnt = 0; cnt < lb->bor_nodes[proc].size(); cnt++) { |
| 402 |
for (size_t cnt2 = 0; cnt2 < lb->born_procs[proc][cnt].size(); cnt2++) { |
| 403 |
n_cmap_nodes[cnt3] = lb->bor_nodes[proc][cnt]; |
| 404 |
n_cmap_procs[cnt3++] = lb->born_procs[proc][cnt][cnt2]; |
| 405 |
} |
| 406 |
} |
| 407 |
|
| 408 |
/* |
| 409 |
* Reorder the nodal communication maps so that they are ordered |
| 410 |
* by processor and then by global ID. |
| 411 |
*/ |
| 412 |
/* This is a 2-key sort */ |
| 413 |
qsort2(Data(n_cmap_procs), Data(n_cmap_nodes), cnt3); |
| 414 |
|
| 415 |
/* Output the nodal communication map */ |
| 416 |
if (ex_put_node_cmap(exoid, 1, Data(n_cmap_nodes), Data(n_cmap_procs), proc) < 0) { |
| 417 |
Gen_Error(0, "fatal: unable to output nodal communication map"); |
| 418 |
return 0; |
| 419 |
} |
| 420 |
} /* End "if (nsize > 0)" */ |
| 421 |
|
| 422 |
/* Output the elemental communication map */ |
| 423 |
if (!lb->e_cmap_elems[proc].empty()) { |
| 424 |
if (ex_put_elem_cmap(exoid, 1, Data(lb->e_cmap_elems[proc]), Data(lb->e_cmap_sides[proc]), |
| 425 |
Data(lb->e_cmap_procs[proc]), proc) < 0) { |
| 426 |
Gen_Error(0, "fatal: unable to output elemental communication map"); |
| 427 |
return 0; |
| 428 |
} |
| 429 |
} |
| 430 |
|
| 431 |
} /* End "for(proc=0; proc < machine->num_procs; proc++)" */ |
| 432 |
} |
| 433 |
return 1; |
| 434 |
} /*------------------------End write_nemesis()------------------------------*/ |
| 435 |
|
| 436 |
/*****************************************************************************/ |
| 437 |
/*****************************************************************************/ |
| 438 |
/*****************************************************************************/ |
| 439 |
/* This function outputs an ExodusII file for the purposes of visualizing |
| 440 |
* the load balance. |
| 441 |
*****************************************************************************/ |
| 442 |
template int write_vis(std::string &nemI_out_file, std::string &exoII_inp_file, |
| 443 |
Machine_Description *machine, Problem_Description *prob, |
| 444 |
Mesh_Description<int> *mesh, LB_Description<int> *lb); |
| 445 |
template int write_vis(std::string &nemI_out_file, std::string &exoII_inp_file, |
| 446 |
Machine_Description *machine, Problem_Description *prob, |
| 447 |
Mesh_Description<int64_t> *mesh, LB_Description<int64_t> *lb); |
| 448 |
|
| 449 |
template <typename INT> |
| 450 |
int write_vis(std::string &nemI_out_file, std::string &exoII_inp_file, Machine_Description *machine, |
| 451 |
Problem_Description *prob, Mesh_Description<INT> *mesh, LB_Description<INT> *lb) |
| 452 |
{ |
| 453 |
/* Generate the file name for the visualization file */ |
| 454 |
std::string vis_file_name = remove_extension(nemI_out_file); |
| 455 |
vis_file_name += "-vis.exoII"; |
| 456 |
|
| 457 |
/* Generate the title for the file */ |
| 458 |
std::string title = fmt::format("{} {} load balance visualization file", UTIL_NAME, ELB_VERSION); |
| 459 |
|
| 460 |
/* |
| 461 |
* If the vis technique is to be by element block then calculate the |
| 462 |
* number of element blocks. |
| 463 |
*/ |
| 464 |
int vis_nelem_blks; |
| 465 |
if (prob->type == DecompType::ELEMENTAL) { |
| 466 |
vis_nelem_blks = machine->num_procs; |
| 467 |
} |
| 468 |
else { |
| 469 |
vis_nelem_blks = machine->num_procs + 1; |
| 470 |
} |
| 471 |
|
| 472 |
/* Create the ExodusII file */ |
| 473 |
fmt::print("Outputting load balance visualization file {}\n", vis_file_name); |
| 474 |
int cpu_ws = 0; |
| 475 |
int io_ws = 0; |
| 476 |
int mode = EX_CLOBBER; |
| 477 |
if (prob->int64db | prob->int64api) { |
| 478 |
mode |= EX_NETCDF4 | EX_NOCLASSIC | prob->int64db | prob->int64api; |
| 479 |
} |
| 480 |
int exid_vis; |
| 481 |
if ((exid_vis = ex_create(vis_file_name.c_str(), mode, &cpu_ws, &io_ws)) < 0) { |
| 482 |
Gen_Error(0, "fatal: unable to create visualization output file"); |
| 483 |
return 0; |
| 484 |
} |
| 485 |
ON_BLOCK_EXIT(ex_close, exid_vis); |
| 486 |
|
| 487 |
/* |
| 488 |
* Open the original input ExodusII file, read the values for the |
| 489 |
* element blocks and output them to the visualization file. |
| 490 |
*/ |
| 491 |
int icpu_ws = 0; |
| 492 |
int iio_ws = 0; |
| 493 |
float vers = 0.0; |
| 494 |
mode = EX_READ | prob->int64api; |
| 495 |
int exid_inp; |
| 496 |
if ((exid_inp = ex_open(exoII_inp_file.c_str(), mode, &icpu_ws, &iio_ws, &vers)) < 0) { |
| 497 |
Gen_Error(0, "fatal: unable to open input ExodusII file"); |
| 498 |
return 0; |
| 499 |
} |
| 500 |
ON_BLOCK_EXIT(ex_close, exid_inp); |
| 501 |
|
| 502 |
std::vector<std::string> elem_type(mesh->num_el_blks); |
| 503 |
std::vector<INT> el_blk_ids(mesh->num_el_blks); |
| 504 |
std::vector<INT> el_cnt_blk(mesh->num_el_blks); |
| 505 |
std::vector<INT> node_pel_blk(mesh->num_el_blks); |
| 506 |
std::vector<INT> nattr_el_blk(mesh->num_el_blks); |
| 507 |
|
| 508 |
if (ex_get_ids(exid_inp, EX_ELEM_BLOCK, Data(el_blk_ids)) < 0) { |
| 509 |
Gen_Error(0, "fatal: unable to get element block IDs"); |
| 510 |
return 0; |
| 511 |
} |
| 512 |
|
| 513 |
int acc_vis = ELB_TRUE; // Output a different element block per processor |
| 514 |
if (prob->vis_out == 2) { |
| 515 |
acc_vis = ELB_FALSE; // Output a nodal/element variable showing processor |
| 516 |
} |
| 517 |
|
| 518 |
size_t nsize = 0; |
| 519 |
|
| 520 |
/* |
| 521 |
* Find out if the mesh consists of mixed elements. If not then |
| 522 |
* element blocks will be used to visualize the partitioning. Otherwise |
| 523 |
* nodal/element results will be used. |
| 524 |
*/ |
| 525 |
for (size_t ecnt = 0; ecnt < mesh->num_el_blks; ecnt++) { |
| 526 |
char type[MAX_STR_LENGTH + 1]; |
| 527 |
if (ex_get_block(exid_inp, EX_ELEM_BLOCK, el_blk_ids[ecnt], type, &el_cnt_blk[ecnt], |
| 528 |
&node_pel_blk[ecnt], nullptr, nullptr, &nattr_el_blk[ecnt]) < 0) { |
| 529 |
Gen_Error(0, "fatal: unable to get element block parameters"); |
| 530 |
return 0; |
| 531 |
} |
| 532 |
elem_type[ecnt] = type; |
| 533 |
nsize += el_cnt_blk[ecnt] * node_pel_blk[ecnt]; |
| 534 |
|
| 535 |
if (elem_type[0] == elem_type[ecnt]) { |
| 536 |
if (node_pel_blk[0] != node_pel_blk[ecnt]) { |
| 537 |
acc_vis = ELB_FALSE; |
| 538 |
} |
| 539 |
} |
| 540 |
else { |
| 541 |
acc_vis = ELB_FALSE; |
| 542 |
} |
| 543 |
} |
| 544 |
|
| 545 |
if (acc_vis == ELB_TRUE) { |
| 546 |
/* Output the initial information */ |
| 547 |
if (ex_put_init(exid_vis, title.c_str(), mesh->num_dims, mesh->num_nodes, mesh->num_elems, |
| 548 |
vis_nelem_blks, 0, 0) < 0) { |
| 549 |
Gen_Error(0, "fatal: unable to output initial params to vis file"); |
| 550 |
return 0; |
| 551 |
} |
| 552 |
|
| 553 |
/* Output the nodal coordinates */ |
| 554 |
float *xptr = nullptr; |
| 555 |
float *yptr = nullptr; |
| 556 |
float *zptr = nullptr; |
| 557 |
switch (mesh->num_dims) { |
| 558 |
case 3: zptr = Data(mesh->coords) + 2 * mesh->num_nodes; FALL_THROUGH; |
| 559 |
case 2: yptr = Data(mesh->coords) + mesh->num_nodes; FALL_THROUGH; |
| 560 |
case 1: xptr = Data(mesh->coords); |
| 561 |
} |
| 562 |
if (ex_put_coord(exid_vis, xptr, yptr, zptr) < 0) { |
| 563 |
Gen_Error(0, "fatal: unable to output coords to vis file"); |
| 564 |
return 0; |
| 565 |
} |
| 566 |
|
| 567 |
const char *coord_names[] = {"X", "Y", "Z"}; |
| 568 |
if (ex_put_coord_names(exid_vis, (char **)coord_names) < 0) { |
| 569 |
Gen_Error(0, "fatal: unable to output coordinate names"); |
| 570 |
return 0; |
| 571 |
} |
| 572 |
|
| 573 |
std::vector<INT> elem_block(mesh->num_elems); |
| 574 |
std::vector<INT> elem_map(mesh->num_elems); |
| 575 |
std::vector<INT> tmp_connect(nsize); |
| 576 |
for (size_t ecnt = 0; ecnt < mesh->num_elems; ecnt++) { |
| 577 |
elem_map[ecnt] = ecnt + 1; |
| 578 |
if (prob->type == DecompType::ELEMENTAL) { |
| 579 |
elem_block[ecnt] = lb->vertex2proc[ecnt]; |
| 580 |
} |
| 581 |
else { |
| 582 |
int proc = lb->vertex2proc[mesh->connect[ecnt][0]]; |
| 583 |
int nnodes = get_elem_info(ElementInfo::NNODES, mesh->elem_type[ecnt]); |
| 584 |
elem_block[ecnt] = proc; |
| 585 |
for (int ncnt = 1; ncnt < nnodes; ncnt++) { |
| 586 |
if (lb->vertex2proc[mesh->connect[ecnt][ncnt]] != proc) { |
| 587 |
elem_block[ecnt] = machine->num_procs; |
| 588 |
break; |
| 589 |
} |
| 590 |
} |
| 591 |
} |
| 592 |
} |
| 593 |
|
| 594 |
int ccnt = 0; |
| 595 |
std::vector<INT> vis_el_blk_ptr(vis_nelem_blks + 1); |
| 596 |
for (INT bcnt = 0; bcnt < vis_nelem_blks; bcnt++) { |
| 597 |
vis_el_blk_ptr[bcnt] = ccnt; |
| 598 |
int pos = 0; |
| 599 |
int old_pos = 0; |
| 600 |
INT *el_ptr = Data(elem_block); |
| 601 |
size_t ecnt = mesh->num_elems; |
| 602 |
while (pos != -1) { |
| 603 |
pos = in_list(bcnt, ecnt, el_ptr); |
| 604 |
if (pos != -1) { |
| 605 |
old_pos += pos + 1; |
| 606 |
ecnt = mesh->num_elems - old_pos; |
| 607 |
el_ptr = Data(elem_block) + old_pos; |
| 608 |
int nnodes = get_elem_info(ElementInfo::NNODES, mesh->elem_type[old_pos - 1]); |
| 609 |
for (int ncnt = 0; ncnt < nnodes; ncnt++) { |
| 610 |
tmp_connect[ccnt++] = mesh->connect[old_pos - 1][ncnt] + 1; |
| 611 |
} |
| 612 |
} |
| 613 |
} |
| 614 |
} |
| 615 |
vis_el_blk_ptr[vis_nelem_blks] = ccnt; |
| 616 |
|
| 617 |
/* Output the element map */ |
| 618 |
if (ex_put_map(exid_vis, Data(elem_map)) < 0) { |
| 619 |
Gen_Error(0, "fatal: unable to output element number map"); |
| 620 |
return 0; |
| 621 |
} |
| 622 |
|
| 623 |
/* Output the visualization element blocks */ |
| 624 |
for (int bcnt = 0; bcnt < vis_nelem_blks; bcnt++) { |
| 625 |
/* |
| 626 |
* Note this assumes all the blocks contain the same type |
| 627 |
* element. |
| 628 |
*/ |
| 629 |
int ecnt = (vis_el_blk_ptr[bcnt + 1] - vis_el_blk_ptr[bcnt]) / node_pel_blk[0]; |
| 630 |
if (ex_put_block(exid_vis, EX_ELEM_BLOCK, bcnt + 1, elem_type[0].c_str(), ecnt, |
| 631 |
node_pel_blk[0], 0, 0, 0) < 0) { |
| 632 |
Gen_Error(0, "fatal: unable to output element block params"); |
| 633 |
return 0; |
| 634 |
} |
| 635 |
|
| 636 |
/* Output the connectivity */ |
| 637 |
if (ex_put_conn(exid_vis, EX_ELEM_BLOCK, bcnt + 1, &tmp_connect[vis_el_blk_ptr[bcnt]], |
| 638 |
nullptr, nullptr) < 0) { |
| 639 |
Gen_Error(0, "fatal: unable to output element connectivity"); |
| 640 |
return 0; |
| 641 |
} |
| 642 |
} |
| 643 |
} |
| 644 |
|
| 645 |
else { /* For nodal/element results visualization of the partitioning. */ |
| 646 |
// Copy the mesh portion to the vis file. |
| 647 |
ex_copy(exid_inp, exid_vis); |
| 648 |
|
| 649 |
/* Set up the file for nodal/element results */ |
| 650 |
float time_val = 0.0; |
| 651 |
if (ex_put_time(exid_vis, 1, &time_val) < 0) { |
| 652 |
Gen_Error(0, "fatal: unable to output time to vis file"); |
| 653 |
return 0; |
| 654 |
} |
| 655 |
|
| 656 |
const char *var_names[] = {"proc"}; |
| 657 |
if (prob->type == DecompType::NODAL) { |
| 658 |
/* Allocate memory for the nodal values */ |
| 659 |
std::vector<float> proc_vals(mesh->num_nodes); |
| 660 |
|
| 661 |
if (ex_put_variable_param(exid_vis, EX_NODAL, 1) < 0) { |
| 662 |
Gen_Error(0, "fatal: unable to output var params to vis file"); |
| 663 |
return 0; |
| 664 |
} |
| 665 |
|
| 666 |
if (ex_put_variable_names(exid_vis, EX_NODAL, 1, (char **)var_names) < 0) { |
| 667 |
Gen_Error(0, "fatal: unable to output variable name"); |
| 668 |
return 0; |
| 669 |
} |
| 670 |
|
| 671 |
/* Do some problem specific assignment */ |
| 672 |
for (size_t ncnt = 0; ncnt < mesh->num_nodes; ncnt++) { |
| 673 |
proc_vals[ncnt] = lb->vertex2proc[ncnt]; |
| 674 |
} |
| 675 |
|
| 676 |
for (int pcnt = 0; pcnt < machine->num_procs; pcnt++) { |
| 677 |
for (const auto &elem : lb->bor_nodes[pcnt]) { |
| 678 |
proc_vals[elem] = machine->num_procs + 1; |
| 679 |
} |
| 680 |
} |
| 681 |
|
| 682 |
/* Output the nodal variables */ |
| 683 |
if (ex_put_var(exid_vis, 1, EX_NODAL, 1, 1, mesh->num_nodes, Data(proc_vals)) < 0) { |
| 684 |
Gen_Error(0, "fatal: unable to output nodal variables"); |
| 685 |
return 0; |
| 686 |
} |
| 687 |
} |
| 688 |
else if (prob->type == DecompType::ELEMENTAL) { |
| 689 |
/* Allocate memory for the element values */ |
| 690 |
std::vector<float> proc_vals(mesh->num_elems); |
| 691 |
|
| 692 |
if (ex_put_variable_param(exid_vis, EX_ELEM_BLOCK, 1) < 0) { |
| 693 |
Gen_Error(0, "fatal: unable to output var params to vis file"); |
| 694 |
return 0; |
| 695 |
} |
| 696 |
|
| 697 |
if (ex_put_variable_names(exid_vis, EX_ELEM_BLOCK, 1, (char **)var_names) < 0) { |
| 698 |
Gen_Error(0, "fatal: unable to output variable name"); |
| 699 |
return 0; |
| 700 |
} |
| 701 |
|
| 702 |
/* Do some problem specific assignment */ |
| 703 |
for (int proc = 0; proc < machine->num_procs; proc++) { |
| 704 |
for (size_t e = 0; e < lb->int_elems[proc].size(); e++) { |
| 705 |
size_t ecnt = lb->int_elems[proc][e]; |
| 706 |
proc_vals[ecnt] = proc; |
| 707 |
} |
| 708 |
|
| 709 |
for (size_t e = 0; e < lb->bor_elems[proc].size(); e++) { |
| 710 |
size_t ecnt = lb->bor_elems[proc][e]; |
| 711 |
proc_vals[ecnt] = proc; |
| 712 |
} |
| 713 |
} |
| 714 |
|
| 715 |
/* Output the element variables */ |
| 716 |
size_t offset = 0; |
| 717 |
for (size_t i = 0; i < mesh->num_el_blks; i++) { |
| 718 |
if (ex_put_var(exid_vis, 1, EX_ELEM_BLOCK, 1, el_blk_ids[i], el_cnt_blk[i], |
| 719 |
&proc_vals[offset]) < 0) { |
| 720 |
Gen_Error(0, "fatal: unable to output nodal variables"); |
| 721 |
return 0; |
| 722 |
} |
| 723 |
offset += el_cnt_blk[i]; |
| 724 |
} |
| 725 |
} |
| 726 |
} |
| 727 |
return 1; |
| 728 |
} /*---------------------------End write_vis()-------------------------------*/ |