| 1 |
/* |
| 2 |
* Copyright(C) 1999-2025 National Technology & Engineering Solutions |
| 3 |
* of Sandia, LLC (NTESS). Under the terms of Contract DE-NA0003525 with |
| 4 |
* NTESS, the U.S. Government retains certain rights in this software. |
| 5 |
* |
| 6 |
* See packages/seacas/LICENSE for details |
| 7 |
*/ |
| 8 |
#include "copy_string_cpp.h" |
| 9 |
#include "el_check_monot.h" // for check_monot |
| 10 |
#include "el_elm.h" // for HEXSHELL, NN_SIDE, etc |
| 11 |
#include "exodusII.h" // for ex_inquire_int, etc |
| 12 |
#include "fmt/ostream.h" |
| 13 |
#include "nem_spread.h" // for NemSpread, second, etc |
| 14 |
#include "netcdf.h" // for nc_set_fill, NC_NOFILL |
| 15 |
#include "pe_common.h" // for MAX_CHUNK_SIZE |
| 16 |
#include "pe_str_util_const.h" // for string_to_lower |
| 17 |
#include "ps_pario_const.h" // for PIO_Time_Array, etc |
| 18 |
#include "rf_allo.h" // for safe_free, array_alloc |
| 19 |
#include "rf_io_const.h" // for Debug_Flag, etc |
| 20 |
#include "rf_util.h" // for print_line, my_sort |
| 21 |
#include "sort_utils.h" // for gds_qsort |
| 22 |
#include "vector_data.h" |
| 23 |
#include <cassert> // for assert |
| 24 |
#include <climits> // for INT_MAX |
| 25 |
#include <cstddef> // for size_t |
| 26 |
#include <cstdio> // for stderr |
| 27 |
#include <cstdlib> // for exit, free |
| 28 |
#include <string> // for string |
| 29 |
#include <vector> // for vector |
| 30 |
template <typename T, typename INT> class Globals; |
| 31 |
|
| 32 |
/*****************************************************************************/ |
| 33 |
/*****************************************************************************/ |
| 34 |
/*****************************************************************************/ |
| 35 |
namespace { |
| 36 |
template <typename T, typename INT> |
| 37 |
int check_sizes(int num_proc, std::vector<INT> &Elem_Blk_Ids, std::vector<INT> &Node_Set_Ids, |
| 38 |
std::vector<INT> &Side_Set_Ids, Globals<T, INT> &globals) |
| 39 |
{ |
| 40 |
/* Do we need 64-bit to write the entity (block/set) ids? */ |
| 41 |
bool ids64bit = false; |
| 42 |
if (!Elem_Blk_Ids.empty()) { |
| 43 |
for (int i = 0; i < globals.Num_Elem_Blk && !ids64bit; i++) { |
| 44 |
if (Elem_Blk_Ids[i] >= INT_MAX) { |
| 45 |
ids64bit = true; |
| 46 |
} |
| 47 |
} |
| 48 |
} |
| 49 |
if (!Node_Set_Ids.empty()) { |
| 50 |
for (int i = 0; i < globals.Num_Node_Set && !ids64bit; i++) { |
| 51 |
if (Node_Set_Ids[i] >= INT_MAX) { |
| 52 |
ids64bit = true; |
| 53 |
} |
| 54 |
} |
| 55 |
} |
| 56 |
if (!Side_Set_Ids.empty()) { |
| 57 |
for (int i = 0; i < globals.Num_Side_Set && !ids64bit; i++) { |
| 58 |
if (Side_Set_Ids[i] >= INT_MAX) { |
| 59 |
ids64bit = true; |
| 60 |
} |
| 61 |
} |
| 62 |
} |
| 63 |
|
| 64 |
/* Do we need 64-bit to write the global node/element maps? */ |
| 65 |
bool global_counts_64bit = false; |
| 66 |
if (globals.Num_Node >= INT_MAX) { |
| 67 |
global_counts_64bit = true; |
| 68 |
} |
| 69 |
if (globals.Num_Elem >= INT_MAX) { |
| 70 |
global_counts_64bit = true; |
| 71 |
} |
| 72 |
|
| 73 |
/* Does any individual mesh have > INT_MAX nodes/elements? */ |
| 74 |
bool local_counts_64bit = false; |
| 75 |
for (int iproc = 0; iproc < num_proc && !local_counts_64bit; iproc++) { |
| 76 |
if (globals.Num_Internal_Nodes[iproc] + globals.Num_Border_Nodes[iproc] + |
| 77 |
globals.Num_External_Nodes[iproc] >= |
| 78 |
INT_MAX) { |
| 79 |
local_counts_64bit = true; |
| 80 |
} |
| 81 |
if (globals.Num_Internal_Elems[iproc] + globals.Num_Border_Elems[iproc] >= INT_MAX) { |
| 82 |
local_counts_64bit = true; |
| 83 |
} |
| 84 |
} |
| 85 |
|
| 86 |
int mode = 0; |
| 87 |
if (ids64bit) { |
| 88 |
mode |= EX_IDS_INT64_DB; |
| 89 |
fmt::print("-- ID output requires 64-bit integers.\n"); |
| 90 |
} |
| 91 |
|
| 92 |
if (global_counts_64bit) { |
| 93 |
mode |= EX_MAPS_INT64_DB; |
| 94 |
fmt::print("-- Global Id map output requires 64-bit integers.\n"); |
| 95 |
} |
| 96 |
|
| 97 |
if (local_counts_64bit) { |
| 98 |
mode |= EX_BULK_INT64_DB; |
| 99 |
fmt::print("-- Bulk data output requires 64-bit integers.\n"); |
| 100 |
} |
| 101 |
|
| 102 |
return mode; |
| 103 |
} |
| 104 |
|
| 105 |
void find_message_info(size_t iunit_size, size_t max_units, size_t *num_units_per_message, |
| 106 |
size_t *num_messages, size_t *num_units_left_over) |
| 107 |
{ |
| 108 |
// Function which determines information regarding the size and number of |
| 109 |
// messages to be used by the functions which broadcast the mesh information. |
| 110 |
|
| 111 |
if (iunit_size > 0) { |
| 112 |
*num_units_per_message = MAX_CHUNK_SIZE / (2 * iunit_size); |
| 113 |
} |
| 114 |
else { |
| 115 |
fmt::print(stderr, "ERROR: find_message_info called with unit_size = 0.\n"); |
| 116 |
exit(1); |
| 117 |
} |
| 118 |
|
| 119 |
if (*num_units_per_message > max_units) { |
| 120 |
*num_units_per_message = max_units; |
| 121 |
} |
| 122 |
|
| 123 |
*num_messages = max_units / (*num_units_per_message); |
| 124 |
*num_units_left_over = max_units - *num_messages * (*num_units_per_message); |
| 125 |
|
| 126 |
if (max_units % (*num_units_per_message) != 0) { |
| 127 |
(*num_messages)++; |
| 128 |
} |
| 129 |
} |
| 130 |
} // namespace |
| 131 |
|
| 132 |
template void NemSpread<double, int>::load_mesh(); |
| 133 |
template void NemSpread<float, int>::load_mesh(); |
| 134 |
template void NemSpread<double, int64_t>::load_mesh(); |
| 135 |
template void NemSpread<float, int64_t>::load_mesh(); |
| 136 |
|
| 137 |
template <typename T, typename INT> void NemSpread<T, INT>::load_mesh() |
| 138 |
{ |
| 139 |
|
| 140 |
/* Function which reads the EXODUS database which contains the mesh. It |
| 141 |
* does this under the control of another EXODUS database file which |
| 142 |
* contains the load-balance information. The load balance information must |
| 143 |
* have been already read. |
| 144 |
* |
| 145 |
*---------------------------------------------------------------------------- |
| 146 |
* |
| 147 |
* Functions called: |
| 148 |
* |
| 149 |
* check_exodus_error -- function which handles the error code returned by |
| 150 |
* calls to EXODUS API routines. |
| 151 |
* construct_lb_filename -- function which appends the string '-#' where |
| 152 |
* '#' is the number of processors to the name of |
| 153 |
* the EXODUS filename. |
| 154 |
* read_lb -- function which reads the load-balance information |
| 155 |
* from an EXODUS database for a given processor. |
| 156 |
* read_coord -- function which reads the nodal coordinates information |
| 157 |
* from an EXODUS database for a given processor. |
| 158 |
* read_elem_blk_ids-- Function which read the element block ids and other |
| 159 |
* global information specific to the element block level. |
| 160 |
* read_elem_blk -- Function which reads the element level information |
| 161 |
* for each element block in the mesh |
| 162 |
* read_node_sets -- function which reads the node sets information from an |
| 163 |
* EXODUS database for a given processor. |
| 164 |
* read_side_sets -- function which reads the side sets information from an |
| 165 |
* EXODUS database for a given processor. |
| 166 |
* |
| 167 |
*---------------------------------------------------------------------------- |
| 168 |
*/ |
| 169 |
|
| 170 |
/* Local variables */ |
| 171 |
|
| 172 |
int mesh_exoid = 0; |
| 173 |
int cpu_ws; |
| 174 |
float version; |
| 175 |
double start_time = 0.0; |
| 176 |
|
| 177 |
/* Allocate some memory for each processor read by this processor */ |
| 178 |
globals.Proc_Num_Elem_Blk = (int *)array_alloc(__FILE__, __LINE__, 1, Proc_Info[2], sizeof(int)); |
| 179 |
globals.Proc_Num_Node_Sets = (int *)array_alloc(__FILE__, __LINE__, 1, Proc_Info[2], sizeof(int)); |
| 180 |
globals.Proc_Num_Side_Sets = (int *)array_alloc(__FILE__, __LINE__, 1, Proc_Info[2], sizeof(int)); |
| 181 |
|
| 182 |
globals.Proc_NS_List_Length = |
| 183 |
(INT *)array_alloc(__FILE__, __LINE__, 1, Proc_Info[2], sizeof(INT)); |
| 184 |
globals.Proc_SS_Elem_List_Length = |
| 185 |
(INT *)array_alloc(__FILE__, __LINE__, 1, Proc_Info[2], sizeof(INT)); |
| 186 |
|
| 187 |
/* Initialize */ |
| 188 |
for (int iproc = 0; iproc < Proc_Info[2]; iproc++) { |
| 189 |
globals.Proc_Num_Elem_Blk[iproc] = 0; |
| 190 |
globals.Proc_Num_Node_Sets[iproc] = 0; |
| 191 |
globals.Proc_Num_Side_Sets[iproc] = 0; |
| 192 |
globals.Proc_NS_List_Length[iproc] = 0; |
| 193 |
globals.Proc_SS_Elem_List_Length[iproc] = 0; |
| 194 |
} |
| 195 |
|
| 196 |
globals.GElem_Blks = (INT **)array_alloc(__FILE__, __LINE__, 1, 24 * Proc_Info[2], sizeof(INT *)); |
| 197 |
globals.Proc_Nodes_Per_Elem = globals.GElem_Blks + Proc_Info[2]; |
| 198 |
globals.Proc_Elem_Blk_Ids = globals.Proc_Nodes_Per_Elem + Proc_Info[2]; |
| 199 |
globals.Proc_Elem_Blk_Types = globals.Proc_Elem_Blk_Ids + Proc_Info[2]; |
| 200 |
globals.Proc_Num_Attr = globals.Proc_Elem_Blk_Types + Proc_Info[2]; |
| 201 |
globals.Proc_Num_Elem_In_Blk = globals.Proc_Num_Attr + Proc_Info[2]; |
| 202 |
globals.Proc_Connect_Ptr = globals.Proc_Num_Elem_In_Blk + Proc_Info[2]; |
| 203 |
globals.Proc_Elem_Connect = globals.Proc_Connect_Ptr + Proc_Info[2]; |
| 204 |
globals.Proc_NS_Ids = globals.Proc_Elem_Connect + Proc_Info[2]; |
| 205 |
globals.Proc_NS_Count = globals.Proc_NS_Ids + Proc_Info[2]; |
| 206 |
globals.Proc_NS_DF_Count = globals.Proc_NS_Count + Proc_Info[2]; |
| 207 |
globals.Proc_NS_Pointers = globals.Proc_NS_DF_Count + Proc_Info[2]; |
| 208 |
globals.Proc_NS_List = globals.Proc_NS_Pointers + Proc_Info[2]; |
| 209 |
globals.GNode_Sets = globals.Proc_NS_List + Proc_Info[2]; |
| 210 |
globals.Proc_NS_GNMap_List = globals.GNode_Sets + Proc_Info[2]; |
| 211 |
globals.Proc_SS_Ids = globals.Proc_NS_GNMap_List + Proc_Info[2]; |
| 212 |
globals.Proc_SS_Elem_Count = globals.Proc_SS_Ids + Proc_Info[2]; |
| 213 |
globals.Proc_SS_DF_Count = globals.Proc_SS_Elem_Count + Proc_Info[2]; |
| 214 |
globals.Proc_SS_Elem_Pointers = globals.Proc_SS_DF_Count + Proc_Info[2]; |
| 215 |
globals.Proc_SS_Elem_List = globals.Proc_SS_Elem_Pointers + Proc_Info[2]; |
| 216 |
globals.Proc_SS_Side_List = globals.Proc_SS_Elem_List + Proc_Info[2]; |
| 217 |
globals.Proc_SS_DF_Pointers = globals.Proc_SS_Side_List + Proc_Info[2]; |
| 218 |
globals.GSide_Sets = globals.Proc_SS_DF_Pointers + Proc_Info[2]; |
| 219 |
globals.Proc_SS_GEMap_List = globals.GSide_Sets + Proc_Info[2]; |
| 220 |
|
| 221 |
globals.Proc_Global_Elem_Id_Map.resize(Proc_Info[2]); |
| 222 |
globals.Proc_Global_Node_Id_Map.resize(Proc_Info[2]); |
| 223 |
|
| 224 |
/* Initialize */ |
| 225 |
for (int iproc = 0; iproc < 24 * Proc_Info[2]; iproc++) { |
| 226 |
globals.GElem_Blks[iproc] = nullptr; |
| 227 |
} |
| 228 |
|
| 229 |
globals.Coor = (T ***)array_alloc(__FILE__, __LINE__, 1, Proc_Info[2], sizeof(T **)); |
| 230 |
|
| 231 |
/* Initialize */ |
| 232 |
for (int iproc = 0; iproc < Proc_Info[2]; iproc++) { |
| 233 |
globals.Coor[iproc] = nullptr; |
| 234 |
} |
| 235 |
|
| 236 |
globals.Proc_Elem_Attr.resize(Proc_Info[2]); |
| 237 |
globals.Proc_NS_Dist_Fact.resize(Proc_Info[2]); |
| 238 |
globals.Proc_SS_Dist_Fact.resize(Proc_Info[2]); |
| 239 |
|
| 240 |
/* Check for a problem which has too many processors for a given mesh */ |
| 241 |
|
| 242 |
if (globals.Num_Node / Proc_Info[0] < 1) { |
| 243 |
fmt::print(stderr, |
| 244 |
"[{}]: ERROR: Problem divided among too many " |
| 245 |
"processors.\n", |
| 246 |
__func__); |
| 247 |
exit(1); |
| 248 |
} |
| 249 |
else if (globals.Num_Elem / Proc_Info[0] < 1) { |
| 250 |
fmt::print(stderr, |
| 251 |
"[{}]: ERROR: Problem divided among too many " |
| 252 |
"processors.\n", |
| 253 |
__func__); |
| 254 |
exit(1); |
| 255 |
} |
| 256 |
|
| 257 |
/* Open the EXODUS mesh file */ |
| 258 |
|
| 259 |
/* computing precision should be the same as the database precision |
| 260 |
* |
| 261 |
* EXCEPTION: if the io_ws is smaller than the machine precision, |
| 262 |
* ie - database with io_ws == 4 on a Cray (sizeof(float) == 8), |
| 263 |
* then the cpu_ws must be the machine precision. |
| 264 |
*/ |
| 265 |
{ |
| 266 |
cpu_ws = io_ws; |
| 267 |
int mode = EX_READ | int64api; |
| 268 |
mesh_exoid = ex_open(ExoFile.c_str(), mode, &cpu_ws, &io_ws, &version); |
| 269 |
if (mesh_exoid < 0) { |
| 270 |
fmt::print(stderr, "{}Exodus returned error opening mesh file, {}\n", __func__, ExoFile); |
| 271 |
exit(1); |
| 272 |
} |
| 273 |
} |
| 274 |
mesh_exoid += selected_change_set; |
| 275 |
|
| 276 |
auto max_name_length = ex_inquire_int(mesh_exoid, EX_INQ_DB_MAX_USED_NAME_LENGTH); |
| 277 |
ex_set_max_name_length(mesh_exoid, max_name_length); |
| 278 |
|
| 279 |
globals.Num_QA_Recs = ex_inquire_int(mesh_exoid, EX_INQ_QA); |
| 280 |
|
| 281 |
/* Add a QA record for the spreader */ |
| 282 |
globals.Num_QA_Recs++; |
| 283 |
|
| 284 |
int length_qa = 4 * (globals.Num_QA_Recs); |
| 285 |
globals.QA_Record = |
| 286 |
reinterpret_cast<char **>(array_alloc(__FILE__, __LINE__, 1, length_qa, sizeof(char *))); |
| 287 |
for (int i1 = 0, index = 0; i1 < globals.Num_QA_Recs; i1++) { |
| 288 |
for (int i2 = 0; i2 < 4; i2++) { |
| 289 |
globals.QA_Record[index++] = reinterpret_cast<char *>( |
| 290 |
array_alloc(__FILE__, __LINE__, 1, MAX_STR_LENGTH + 1, sizeof(char))); |
| 291 |
} |
| 292 |
} |
| 293 |
|
| 294 |
if (ex_get_qa(mesh_exoid, (char *(*)[4])globals.QA_Record) < 0) { |
| 295 |
fmt::print(stderr, "[{}]: ERROR, could not get QA record(s)\n", __func__); |
| 296 |
exit(1); |
| 297 |
} |
| 298 |
|
| 299 |
/* Read in the information records */ |
| 300 |
globals.Num_Info_Recs = ex_inquire_int(mesh_exoid, EX_INQ_INFO); |
| 301 |
|
| 302 |
if (globals.Num_Info_Recs > 0) { |
| 303 |
/* Allocate the Information records */ |
| 304 |
globals.Info_Record = (char **)array_alloc(__FILE__, __LINE__, 2, globals.Num_Info_Recs, |
| 305 |
MAX_LINE_LENGTH + 1, sizeof(char)); |
| 306 |
if (!globals.Info_Record) { |
| 307 |
fmt::print(stderr, "[{}]: ERROR, insufficient memory!\n", __func__); |
| 308 |
exit(1); |
| 309 |
} |
| 310 |
|
| 311 |
if (ex_get_info(mesh_exoid, globals.Info_Record) < 0) { |
| 312 |
fmt::print(stderr, "[{}]: ERROR, could not get Info record(s)\n", __func__); |
| 313 |
exit(1); |
| 314 |
} |
| 315 |
} |
| 316 |
|
| 317 |
/* Read in the assembly information */ |
| 318 |
{ |
| 319 |
ex_init_params ex_info; |
| 320 |
ex_get_init_ext(mesh_exoid, &ex_info); |
| 321 |
globals.Num_Assemblies = ex_info.num_assembly; |
| 322 |
} |
| 323 |
|
| 324 |
if (globals.Num_Assemblies > 0) { |
| 325 |
globals.Assemblies.resize(globals.Num_Assemblies); |
| 326 |
for (auto &assembly : globals.Assemblies) { |
| 327 |
assembly.name = nullptr; |
| 328 |
assembly.entity_list = nullptr; |
| 329 |
} |
| 330 |
ex_get_assemblies(mesh_exoid, Data(globals.Assemblies)); |
| 331 |
|
| 332 |
for (auto &assembly : globals.Assemblies) { |
| 333 |
assembly.entity_list = new int64_t[assembly.entity_count]; |
| 334 |
} |
| 335 |
|
| 336 |
// Now get the assembly entity lists... |
| 337 |
ex_get_assemblies(mesh_exoid, Data(globals.Assemblies)); |
| 338 |
} |
| 339 |
|
| 340 |
/* Read in the coordinate frame information */ |
| 341 |
globals.Num_Coordinate_Frames = ex_inquire_int(mesh_exoid, EX_INQ_COORD_FRAMES); |
| 342 |
|
| 343 |
if (globals.Num_Coordinate_Frames > 0) { |
| 344 |
|
| 345 |
/* Allocate the Coordinate Frame records */ |
| 346 |
globals.Coordinate_Frame_Ids = |
| 347 |
(INT *)array_alloc(__FILE__, __LINE__, 1, globals.Num_Coordinate_Frames, sizeof(INT)); |
| 348 |
if (!globals.Coordinate_Frame_Ids) { |
| 349 |
fmt::print(stderr, "[{}]: ERROR, insufficient memory!\n", __func__); |
| 350 |
exit(1); |
| 351 |
} |
| 352 |
|
| 353 |
globals.Coordinate_Frame_Coordinates = |
| 354 |
(T *)array_alloc(__FILE__, __LINE__, 1, 9 * globals.Num_Coordinate_Frames, sizeof(T)); |
| 355 |
if (!globals.Coordinate_Frame_Coordinates) { |
| 356 |
fmt::print(stderr, "[{}]: ERROR, insufficient memory!\n", __func__); |
| 357 |
exit(1); |
| 358 |
} |
| 359 |
|
| 360 |
globals.Coordinate_Frame_Tags = |
| 361 |
(char *)array_alloc(__FILE__, __LINE__, 1, globals.Num_Coordinate_Frames, sizeof(char)); |
| 362 |
if (!globals.Coordinate_Frame_Tags) { |
| 363 |
fmt::print(stderr, "[{}]: ERROR, insufficient memory!\n", __func__); |
| 364 |
exit(1); |
| 365 |
} |
| 366 |
|
| 367 |
int num_frames = 0; |
| 368 |
if (ex_get_coordinate_frames(mesh_exoid, &num_frames, globals.Coordinate_Frame_Ids, |
| 369 |
globals.Coordinate_Frame_Coordinates, |
| 370 |
globals.Coordinate_Frame_Tags) < 0) { |
| 371 |
fmt::print(stderr, "[{}]: ERROR, could not get Coordinate Frame record(s)\n", __func__); |
| 372 |
exit(1); |
| 373 |
} |
| 374 |
if (num_frames != globals.Num_Coordinate_Frames) { |
| 375 |
fmt::print(stderr, "[{}]: ERROR, frame count inconsistency\n", __func__); |
| 376 |
exit(1); |
| 377 |
} |
| 378 |
} |
| 379 |
|
| 380 |
/* |
| 381 |
* Allocate Temporary Arrays that are only used in el_exoII_io.c (Note: Calls |
| 382 |
* to array_alloc are minimized by using pointer arithmetic. Resulting memory |
| 383 |
* structure may be assumed during later broadcast routines) |
| 384 |
*/ |
| 385 |
|
| 386 |
if (globals.Num_Elem_Blk <= 0) { |
| 387 |
fmt::print(stderr, "ERROR, globals.Num_Elem_Blk = {}\n", globals.Num_Elem_Blk); |
| 388 |
exit(1); |
| 389 |
} |
| 390 |
|
| 391 |
Num_Elem_In_Blk.resize(globals.Num_Elem_Blk); |
| 392 |
Num_Nodes_Per_Elem.resize(globals.Num_Elem_Blk); |
| 393 |
Num_Attr_Per_Elem.resize(globals.Num_Elem_Blk); |
| 394 |
Elem_Blk_Ids.resize(globals.Num_Elem_Blk); |
| 395 |
|
| 396 |
Elem_Blk_Types = (char **)array_alloc(__FILE__, __LINE__, 2, globals.Num_Elem_Blk, |
| 397 |
MAX_STR_LENGTH + 1, sizeof(char)); |
| 398 |
Elem_Blk_Names = (char **)array_alloc(__FILE__, __LINE__, 2, globals.Num_Elem_Blk, |
| 399 |
max_name_length + 1, sizeof(char)); |
| 400 |
Elem_Blk_Attr_Names = |
| 401 |
(char ***)array_alloc(__FILE__, __LINE__, 1, globals.Num_Elem_Blk, sizeof(char **)); |
| 402 |
|
| 403 |
Node_Set_Ids.resize(globals.Num_Node_Set); |
| 404 |
std::vector<INT> num_nodes_in_node_set(globals.Num_Node_Set); |
| 405 |
std::vector<INT> num_df_in_nsets(globals.Num_Node_Set); |
| 406 |
|
| 407 |
if (globals.Num_Node_Set > 0) { |
| 408 |
Node_Set_Names = (char **)array_alloc(__FILE__, __LINE__, 2, globals.Num_Node_Set, |
| 409 |
max_name_length + 1, sizeof(char)); |
| 410 |
} |
| 411 |
|
| 412 |
Side_Set_Ids.resize(globals.Num_Side_Set); |
| 413 |
std::vector<INT> num_elem_in_ssets(globals.Num_Side_Set); |
| 414 |
std::vector<INT> num_df_in_ssets(globals.Num_Side_Set); |
| 415 |
|
| 416 |
if (globals.Num_Side_Set > 0) { |
| 417 |
Side_Set_Names = (char **)array_alloc(__FILE__, __LINE__, 2, globals.Num_Side_Set, |
| 418 |
max_name_length + 1, sizeof(char)); |
| 419 |
} |
| 420 |
|
| 421 |
/* |
| 422 |
* Process the Global Element Block IDs and associated information related to |
| 423 |
* element blocks (i.e., read them, broadcast them, and check them against |
| 424 |
* the input file). |
| 425 |
*/ |
| 426 |
start_time = second(); |
| 427 |
|
| 428 |
read_elem_blk_ids(mesh_exoid, max_name_length); |
| 429 |
|
| 430 |
fmt::print("\tTime to read element block IDs: {:.4f}\n", second() - start_time); |
| 431 |
|
| 432 |
/* |
| 433 |
* Process the Node Set IDs and associated information related to node sets |
| 434 |
* (i.e., read them, broadcast them, and check them against the input file). |
| 435 |
*/ |
| 436 |
|
| 437 |
if (globals.Num_Node_Set > 0) { |
| 438 |
start_time = second(); |
| 439 |
read_node_set_ids(mesh_exoid, num_nodes_in_node_set, num_df_in_nsets, max_name_length); |
| 440 |
|
| 441 |
fmt::print("\tTime to read node set IDs: {:.4f}\n", second() - start_time); |
| 442 |
} |
| 443 |
|
| 444 |
/* |
| 445 |
* Process the Side Set IDs and associated information related to side sets |
| 446 |
* (i.e., read them, broadcast them, and check them against the input file). |
| 447 |
*/ |
| 448 |
if (globals.Num_Side_Set > 0) { |
| 449 |
start_time = second(); |
| 450 |
read_side_set_ids(mesh_exoid, num_elem_in_ssets, num_df_in_ssets, max_name_length); |
| 451 |
fmt::print("\tTime to read side set IDs: {:.4f}\n", second() - start_time); |
| 452 |
} |
| 453 |
|
| 454 |
/* |
| 455 |
* Process the element block information. Find out which element blocks have |
| 456 |
* elements needed by which processors. Set-up and fill in vectors which |
| 457 |
* have length equal to the number of element blocks on the processor, |
| 458 |
* globals.Num_Elem_Blk. |
| 459 |
*/ |
| 460 |
start_time = second(); |
| 461 |
|
| 462 |
extract_elem_blk(); |
| 463 |
|
| 464 |
fmt::print("\tTime to extract element block information: {:.4f}\n", second() - start_time); |
| 465 |
|
| 466 |
/* |
| 467 |
* Read the mesh information from the exodus file and broadcast it to the |
| 468 |
* processors. NOTE: this will only read as much information as will fit in |
| 469 |
* the communication buffer. It will then send the information to all the |
| 470 |
* processors so they can extract the information they need. Then, the next |
| 471 |
* block of mesh data will be read and sent, etc., until the entire mesh has |
| 472 |
* been read and distributed to the processors. Note, that the load balance |
| 473 |
* information has already been distributed by this point. Therefore, each |
| 474 |
* node already nodes which global nodes it owns, which elements it has |
| 475 |
* global nodes in that it owns, and which ghost nodes it needs information |
| 476 |
* about. |
| 477 |
* |
| 478 |
* The following information is defined as the mesh information: |
| 479 |
* |
| 480 |
* 1 - Coordinate values of global nodes |
| 481 |
* 2 - Element connectivity graph |
| 482 |
* 3 - Attributes of the elements |
| 483 |
* 4 - Composition of Node Sets |
| 484 |
* 5 - Composition of Side Sets |
| 485 |
*/ |
| 486 |
|
| 487 |
/* Read the coordinates, broadcast, and sort */ |
| 488 |
|
| 489 |
start_time = second(); |
| 490 |
|
| 491 |
read_coord(mesh_exoid, max_name_length); |
| 492 |
|
| 493 |
fmt::print("\tTime to read nodal coordinates: {:.4f}\n", second() - start_time); |
| 494 |
|
| 495 |
/* |
| 496 |
* Read the element connectivity and attributes information. Broadcast it |
| 497 |
* and extract only the information that the current processor needs. |
| 498 |
*/ |
| 499 |
start_time = second(); |
| 500 |
read_elem_blk(mesh_exoid); |
| 501 |
fmt::print("\tTime to read element blocks: {:.4f}\n", second() - start_time); |
| 502 |
|
| 503 |
/* Read the node sets and sort */ |
| 504 |
if (globals.Num_Node_Set > 0) { |
| 505 |
start_time = second(); |
| 506 |
read_node_sets(mesh_exoid, num_nodes_in_node_set, num_df_in_nsets); |
| 507 |
fmt::print("\tTime to read node sets: {:.4f}\n", second() - start_time); |
| 508 |
} |
| 509 |
|
| 510 |
/* Assign the element types. */ |
| 511 |
if (globals.Num_Side_Set > 0) { |
| 512 |
start_time = second(); |
| 513 |
create_elem_types(); /* globals.Elem_Type only used in sideset read */ |
| 514 |
fmt::print("\tTime to categorize element types: {:.4f}\n", second() - start_time); |
| 515 |
|
| 516 |
/* Read the side sets and sort */ |
| 517 |
start_time = second(); |
| 518 |
read_side_sets(mesh_exoid, num_elem_in_ssets, num_df_in_ssets); |
| 519 |
fmt::print("\tTime to read side sets: {:.4f}\n", second() - start_time); |
| 520 |
} |
| 521 |
|
| 522 |
/* Close the EXODUS mesh file */ |
| 523 |
check_exodus_error(ex_close(mesh_exoid), "ex_close"); |
| 524 |
|
| 525 |
/************* Output the information to the parallel disks *****************/ |
| 526 |
/*==========================================================================*/ |
| 527 |
/*==========================================================================*/ |
| 528 |
|
| 529 |
/* Generate the processor to disk map */ |
| 530 |
gen_disk_map(&PIO_Info, Proc_Info, 0, 1); |
| 531 |
|
| 532 |
/* Generate the parallel exodus file name */ |
| 533 |
/* See if any '/' in the name. IF present, isolate the basename of the file */ |
| 534 |
std::string cTemp; |
| 535 |
|
| 536 |
size_t found = Output_File_Base_Name.find_last_of('/'); |
| 537 |
if (found != std::string::npos) { |
| 538 |
/* There is a path separator. Get the portion after the |
| 539 |
* separator |
| 540 |
*/ |
| 541 |
cTemp = Output_File_Base_Name.substr(found + 1); |
| 542 |
} |
| 543 |
else { |
| 544 |
/* No separator; this is already just the basename... */ |
| 545 |
cTemp = Output_File_Base_Name; |
| 546 |
} |
| 547 |
|
| 548 |
if (PIO_Info.Exo_Extension.empty()) { |
| 549 |
cTemp += ".par"; |
| 550 |
} |
| 551 |
else { |
| 552 |
cTemp += PIO_Info.Exo_Extension; |
| 553 |
} |
| 554 |
|
| 555 |
/* Check sizes to see if need to store using 64-bit on the database */ |
| 556 |
int db_mode = check_sizes(Proc_Info[2], Elem_Blk_Ids, Node_Set_Ids, Side_Set_Ids, globals); |
| 557 |
if (force64db) { |
| 558 |
fmt::print("-- Command-line option forcing output to use all 64-bit integers.\n"); |
| 559 |
db_mode |= EX_ALL_INT64_DB; |
| 560 |
} |
| 561 |
|
| 562 |
for (int iproc = Proc_Info[4]; iproc < Proc_Info[4] + Proc_Info[5]; iproc++) { |
| 563 |
std::string Parallel_File_Name = gen_par_filename(cTemp, Proc_Ids[iproc], Proc_Info[0]); |
| 564 |
|
| 565 |
/* Create the parallel Exodus file for writing */ |
| 566 |
if (Debug_Flag >= 7) { |
| 567 |
fmt::print("{}Parallel mesh file name is {}\n", __func__, Parallel_File_Name); |
| 568 |
} |
| 569 |
else { |
| 570 |
if (iproc % 10 == 0 || iproc == Proc_Info[2] - 1) { |
| 571 |
fmt::print("{}", iproc); |
| 572 |
} |
| 573 |
else { |
| 574 |
fmt::print("."); |
| 575 |
} |
| 576 |
} |
| 577 |
|
| 578 |
int mode = EX_CLOBBER; |
| 579 |
mode |= int64api; |
| 580 |
mode |= db_mode; |
| 581 |
if ((mesh_exoid = ex_create(Parallel_File_Name.c_str(), mode, &cpu_ws, &io_ws)) == -1) { |
| 582 |
|
| 583 |
fmt::print(stderr, "[{}] Could not create parallel Exodus file:\n\t{}\n", __func__, |
| 584 |
Parallel_File_Name); |
| 585 |
exit(1); |
| 586 |
} |
| 587 |
|
| 588 |
ex_set_max_name_length(mesh_exoid, max_name_length); |
| 589 |
|
| 590 |
/* Set fill mode off... */ |
| 591 |
{ |
| 592 |
int old_fill; |
| 593 |
nc_set_fill(mesh_exoid, NC_NOFILL, &old_fill); |
| 594 |
} |
| 595 |
|
| 596 |
if (Debug_Flag >= 7) { |
| 597 |
fmt::print("{}Parallel mesh file id is {}\n", __func__, mesh_exoid); |
| 598 |
} |
| 599 |
|
| 600 |
/* Write out a parallel mesh file local to each processor */ |
| 601 |
write_parExo_data(mesh_exoid, max_name_length, iproc, num_nodes_in_node_set, num_elem_in_ssets, |
| 602 |
Num_Elem_In_Blk); |
| 603 |
|
| 604 |
/* Close the parallel exodus file */ |
| 605 |
if (ex_close(mesh_exoid) == -1) { |
| 606 |
fmt::print(stderr, "{}Could not close the parallel Exodus file\n", __func__); |
| 607 |
exit(1); |
| 608 |
} |
| 609 |
} /* End "for(iproc=0; iproc <Proc_Info[2]; iproc++)" */ |
| 610 |
|
| 611 |
if (Debug_Flag >= 4) { |
| 612 |
fmt::print("\n\n\t\tTIMING TABLE FOR PROCESSORS\n"); |
| 613 |
fmt::print("===========================================================\n"); |
| 614 |
fmt::print("[0]: {:6.2f}\n", PIO_Time_Array[0]); |
| 615 |
for (int i1 = 1; i1 < 25; i1++) { |
| 616 |
fmt::print("\t\t[{}]: {:6.2f}\n", i1, PIO_Time_Array[i1]); |
| 617 |
} |
| 618 |
fmt::print("\nOutput rate: {:6.2f} kB/s\n", PIO_Time_Array[25]); |
| 619 |
} |
| 620 |
else if (Debug_Flag >= 1) { |
| 621 |
fmt::print("\n\nOutput rate: {:6.2f} kB/s\n", PIO_Time_Array[25]); |
| 622 |
} |
| 623 |
|
| 624 |
/*--------------------------------------------------------------------------*/ |
| 625 |
/*--------------------------------------------------------------------------*/ |
| 626 |
|
| 627 |
/* Free QA records */ |
| 628 |
for (int i1 = 0; i1 < 4 * (globals.Num_QA_Recs); i1++) { |
| 629 |
safe_free((void **)&(globals.QA_Record[i1])); |
| 630 |
} |
| 631 |
|
| 632 |
safe_free((void **)&globals.QA_Record); |
| 633 |
|
| 634 |
for (int i1 = 0; i1 < globals.Num_Elem_Blk; i1++) { |
| 635 |
if (Num_Attr_Per_Elem[i1] > 0) { |
| 636 |
safe_free((void **)&(Elem_Blk_Attr_Names[i1])); |
| 637 |
} |
| 638 |
} |
| 639 |
safe_free((void **)&Elem_Blk_Attr_Names); |
| 640 |
|
| 641 |
/* Free Coordinate Frames */ |
| 642 |
if (globals.Num_Coordinate_Frames > 0) { |
| 643 |
safe_free((void **)&globals.Coordinate_Frame_Ids); |
| 644 |
safe_free((void **)&globals.Coordinate_Frame_Coordinates); |
| 645 |
safe_free((void **)&globals.Coordinate_Frame_Tags); |
| 646 |
} |
| 647 |
|
| 648 |
// Clear out allocated memory for assembly entity_list... |
| 649 |
for (auto &assembly : globals.Assemblies) { |
| 650 |
free(assembly.name); |
| 651 |
delete[] assembly.entity_list; |
| 652 |
assembly.name = nullptr; |
| 653 |
assembly.entity_list = nullptr; |
| 654 |
} |
| 655 |
|
| 656 |
/* done with the Coordinate names */ |
| 657 |
for (int i1 = 0; i1 < globals.Num_Dim; i1++) { |
| 658 |
safe_free((void **)&(Coord_Name[i1])); |
| 659 |
} |
| 660 |
|
| 661 |
/* Free some local arrays */ |
| 662 |
Num_Elem_In_Blk.clear(); |
| 663 |
Node_Set_Ids.clear(); |
| 664 |
Side_Set_Ids.clear(); |
| 665 |
|
| 666 |
safe_free((void **)&Elem_Blk_Types); |
| 667 |
safe_free((void **)&Node_Set_Names); |
| 668 |
safe_free((void **)&Side_Set_Names); |
| 669 |
safe_free((void **)&Elem_Blk_Names); |
| 670 |
|
| 671 |
safe_free((void **)&Restart_Info.NV_Name); |
| 672 |
safe_free((void **)&Restart_Info.EV_Name); |
| 673 |
safe_free((void **)&Restart_Info.GV_Name); |
| 674 |
safe_free((void **)&Restart_Info.NSV_Name); |
| 675 |
safe_free((void **)&Restart_Info.SSV_Name); |
| 676 |
|
| 677 |
/* |
| 678 |
* free up some other memory so that there is more |
| 679 |
* memory available for reading restart variables |
| 680 |
*/ |
| 681 |
for (int iproc = 0; iproc < Proc_Info[2]; iproc++) { |
| 682 |
safe_free((void **)&(globals.Proc_Connect_Ptr[iproc])); |
| 683 |
safe_free((void **)&(globals.Proc_Elem_Connect[iproc])); |
| 684 |
|
| 685 |
safe_free((void **)&(globals.Coor[iproc])); |
| 686 |
globals.Proc_Elem_Attr[iproc].clear(); |
| 687 |
globals.Proc_NS_Dist_Fact[iproc].clear(); |
| 688 |
globals.Proc_SS_Dist_Fact[iproc].clear(); |
| 689 |
} |
| 690 |
globals.N_Comm_Map.clear(); |
| 691 |
globals.E_Comm_Map.clear(); |
| 692 |
globals.Proc_Elem_Attr.clear(); |
| 693 |
globals.Proc_NS_Dist_Fact.clear(); |
| 694 |
globals.Proc_SS_Dist_Fact.clear(); |
| 695 |
fmt::print("\n"); |
| 696 |
|
| 697 |
} /* END of routine load_mesh () *********************************************/ |
| 698 |
|
| 699 |
/*****************************************************************************/ |
| 700 |
/*****************************************************************************/ |
| 701 |
/*****************************************************************************/ |
| 702 |
|
| 703 |
template <typename T, typename INT> |
| 704 |
void NemSpread<T, INT>::read_elem_blk_ids(int mesh_exoid, int max_name_length) |
| 705 |
{ |
| 706 |
|
| 707 |
/* This function reads part of the element block info from the EXODUS file. |
| 708 |
* It reads all information having a length equal to the number of elements |
| 709 |
* blocks, specifically. |
| 710 |
* |
| 711 |
* The function then broadcasts this information to all processors. |
| 712 |
* |
| 713 |
* The function also calls check_matrl_blks, which checks the element block ids |
| 714 |
* in the input file and in the mesh file for consistency (and prints out a |
| 715 |
* table for a large enough value of Debug_Flag). |
| 716 |
*/ |
| 717 |
|
| 718 |
/* Get the Element Block IDs from the input file */ |
| 719 |
check_exodus_error(ex_get_ids(mesh_exoid, EX_ELEM_BLOCK, Data(Elem_Blk_Ids)), "ex_get_ids"); |
| 720 |
|
| 721 |
check_exodus_error(ex_get_names(mesh_exoid, EX_ELEM_BLOCK, Elem_Blk_Names), "ex_get_names"); |
| 722 |
|
| 723 |
/* |
| 724 |
* Get from the input file: |
| 725 |
* Number of Elements in each element block |
| 726 |
* Number of nodes per element in each element block |
| 727 |
* Number of attributes per element in each element block |
| 728 |
* The element type for elements in each element block |
| 729 |
*/ |
| 730 |
for (int i = 0; i < globals.Num_Elem_Blk; i++) { |
| 731 |
|
| 732 |
check_exodus_error(ex_get_block(mesh_exoid, EX_ELEM_BLOCK, Elem_Blk_Ids[i], Elem_Blk_Types[i], |
| 733 |
&Num_Elem_In_Blk[i], &Num_Nodes_Per_Elem[i], nullptr, nullptr, |
| 734 |
&Num_Attr_Per_Elem[i]), |
| 735 |
"ex_get_elem_block"); |
| 736 |
|
| 737 |
/* Convert element block types to lower case here */ |
| 738 |
string_to_lower(Elem_Blk_Types[i], '\0'); |
| 739 |
|
| 740 |
/* Allocate space for attribute names (if any), read and store. */ |
| 741 |
if (Num_Attr_Per_Elem[i] > 0) { |
| 742 |
Elem_Blk_Attr_Names[i] = (char **)array_alloc(__FILE__, __LINE__, 2, Num_Attr_Per_Elem[i], |
| 743 |
max_name_length + 1, sizeof(char)); |
| 744 |
check_exodus_error( |
| 745 |
ex_get_attr_names(mesh_exoid, EX_ELEM_BLOCK, Elem_Blk_Ids[i], Elem_Blk_Attr_Names[i]), |
| 746 |
"ex_get_attr_names"); |
| 747 |
} |
| 748 |
else { |
| 749 |
Elem_Blk_Attr_Names[i] = nullptr; |
| 750 |
} |
| 751 |
} |
| 752 |
} |
| 753 |
|
| 754 |
/*****************************************************************************/ |
| 755 |
/*****************************************************************************/ |
| 756 |
/*****************************************************************************/ |
| 757 |
|
| 758 |
template <typename T, typename INT> |
| 759 |
void NemSpread<T, INT>::read_node_set_ids(int mesh_exoid, std::vector<INT> &num_nodes_in_node_set, |
| 760 |
std::vector<INT> &num_df_in_nsets, |
| 761 |
int /*max_name_length*/) |
| 762 |
|
| 763 |
/* |
| 764 |
* This function reads part of the node set info from the EXODUS file. It |
| 765 |
* reads all information having a length equal to the number of node sets, |
| 766 |
* specifically. |
| 767 |
* |
| 768 |
* The function then broadcasts this information to all processors. |
| 769 |
* |
| 770 |
*/ |
| 771 |
|
| 772 |
{ |
| 773 |
if (globals.Num_Node_Set > 0) { |
| 774 |
int error = ex_get_ids(mesh_exoid, EX_NODE_SET, Data(Node_Set_Ids)); |
| 775 |
check_exodus_error(error, "ex_get_node_set_ids"); |
| 776 |
|
| 777 |
error = ex_get_names(mesh_exoid, EX_NODE_SET, Node_Set_Names); |
| 778 |
check_exodus_error(error, "ex_get_node_set_ids"); |
| 779 |
|
| 780 |
for (int i = 0; i < globals.Num_Node_Set; i++) { |
| 781 |
check_exodus_error(ex_get_set_param(mesh_exoid, EX_NODE_SET, Node_Set_Ids[i], |
| 782 |
&num_nodes_in_node_set[i], &num_df_in_nsets[i]), |
| 783 |
"ex_get_set_param"); |
| 784 |
} |
| 785 |
} |
| 786 |
|
| 787 |
/* Output debug info */ |
| 788 |
if ((Debug_Flag > 1)) { |
| 789 |
fmt::print("\n\n"); |
| 790 |
print_line("=", 79); |
| 791 |
fmt::print("\tTABLE OF NODE SET ID\'s\n\n"); |
| 792 |
fmt::print("Node_Set_Num ID globals.Num_Nodes\n"); |
| 793 |
print_line("-", 79); |
| 794 |
|
| 795 |
if (globals.Num_Node_Set > 0) { |
| 796 |
for (int i = 0; i < globals.Num_Node_Set; i++) { |
| 797 |
fmt::print("{:6d}{:11d}{:12d}\n", i, Node_Set_Ids[i], num_nodes_in_node_set[i]); |
| 798 |
} |
| 799 |
} |
| 800 |
|
| 801 |
else { |
| 802 |
fmt::print("\tNO NODE SETS ARE DEFINED IN THE MESH FILE\n"); |
| 803 |
} |
| 804 |
print_line("=", 79); |
| 805 |
fmt::print("\n"); |
| 806 |
} |
| 807 |
} |
| 808 |
|
| 809 |
/*****************************************************************************/ |
| 810 |
/*****************************************************************************/ |
| 811 |
/*****************************************************************************/ |
| 812 |
|
| 813 |
template <typename T, typename INT> |
| 814 |
void NemSpread<T, INT>::read_side_set_ids(int mesh_exoid, std::vector<INT> &num_elem_in_ssets, |
| 815 |
std::vector<INT> &num_df_in_ssets, |
| 816 |
int /*max_name_length*/) |
| 817 |
|
| 818 |
/* This function reads part of the side set info from the EXODUS file. It |
| 819 |
* reads all information having a length equal to the number of side sets, |
| 820 |
* specifically. |
| 821 |
* |
| 822 |
* The function then broadcasts this information to all processors. |
| 823 |
* |
| 824 |
*/ |
| 825 |
|
| 826 |
{ |
| 827 |
if (globals.Num_Side_Set > 0) { |
| 828 |
int error = ex_get_ids(mesh_exoid, EX_SIDE_SET, Data(Side_Set_Ids)); |
| 829 |
check_exodus_error(error, "ex_get_side_set_ids"); |
| 830 |
|
| 831 |
error = ex_get_names(mesh_exoid, EX_SIDE_SET, Side_Set_Names); |
| 832 |
check_exodus_error(error, "ex_get_side_set_ids"); |
| 833 |
|
| 834 |
for (int i = 0; i < globals.Num_Side_Set; i++) { |
| 835 |
check_exodus_error(ex_get_set_param(mesh_exoid, EX_SIDE_SET, Side_Set_Ids[i], |
| 836 |
&num_elem_in_ssets[i], &num_df_in_ssets[i]), |
| 837 |
"ex_get_set_param"); |
| 838 |
} |
| 839 |
} |
| 840 |
|
| 841 |
/* Output debug information */ |
| 842 |
if (Debug_Flag > 1) { |
| 843 |
fmt::print("\n\n"); |
| 844 |
print_line("=", 79); |
| 845 |
fmt::print("\tTABLE OF SIDE SET ID\'s\n\n"); |
| 846 |
fmt::print("Side_Set_Num ID Number Elements\n"); |
| 847 |
print_line("-", 79); |
| 848 |
|
| 849 |
if (globals.Num_Side_Set > 0) { |
| 850 |
for (int i = 0; i < globals.Num_Side_Set; i++) { |
| 851 |
fmt::print("{:6d}{:11d} {:12}\n", i, Side_Set_Ids[i], |
| 852 |
fmt::group_digits(num_elem_in_ssets[i])); |
| 853 |
} |
| 854 |
} |
| 855 |
|
| 856 |
else { |
| 857 |
fmt::print("\tNO SIDE SETS ARE DEFINED IN THE MESH FILE\n"); |
| 858 |
} |
| 859 |
print_line("=", 79); |
| 860 |
fmt::print("\n"); |
| 861 |
} |
| 862 |
} |
| 863 |
|
| 864 |
/*****************************************************************************/ |
| 865 |
/*****************************************************************************/ |
| 866 |
/*****************************************************************************/ |
| 867 |
|
| 868 |
template <typename T, typename INT> |
| 869 |
void NemSpread<T, INT>::read_coord(int exoid, int max_name_length) |
| 870 |
{ |
| 871 |
|
| 872 |
/* Function which reads the nodal coordinates information from an |
| 873 |
* EXODUS database for a given processor. |
| 874 |
*/ |
| 875 |
|
| 876 |
/* |
| 877 |
* Calculate the size of the coordinate space for each processors coordinate matrix. |
| 878 |
*/ |
| 879 |
for (int iproc = Proc_Info[4]; iproc < Proc_Info[4] + Proc_Info[5]; iproc++) { |
| 880 |
size_t itotal_nodes = globals.Num_Internal_Nodes[iproc] + globals.Num_Border_Nodes[iproc] + |
| 881 |
globals.Num_External_Nodes[iproc]; |
| 882 |
|
| 883 |
/* Allocate permanent storage for the coordinates */ |
| 884 |
if (itotal_nodes > 0) { |
| 885 |
globals.Coor[iproc] = |
| 886 |
(T **)array_alloc(__FILE__, __LINE__, 2, globals.Num_Dim, itotal_nodes, sizeof(T)); |
| 887 |
} |
| 888 |
else { |
| 889 |
globals.Coor[iproc] = nullptr; |
| 890 |
} |
| 891 |
} |
| 892 |
|
| 893 |
/* Allocate temporary space to hold 1 dimensions worth of coordinates... */ |
| 894 |
std::vector<T> coord(globals.Num_Node); |
| 895 |
|
| 896 |
/* Read in the coordinates and broadcast to the processors */ |
| 897 |
for (int idim = 0; idim < globals.Num_Dim; idim++) { |
| 898 |
switch (idim) { |
| 899 |
case 0: |
| 900 |
check_exodus_error(ex_get_coord(exoid, Data(coord), nullptr, nullptr), "ex_get_coord"); |
| 901 |
break; |
| 902 |
case 1: |
| 903 |
check_exodus_error(ex_get_coord(exoid, nullptr, Data(coord), nullptr), "ex_get_coord"); |
| 904 |
break; |
| 905 |
case 2: |
| 906 |
check_exodus_error(ex_get_coord(exoid, nullptr, nullptr, Data(coord)), "ex_get_coord"); |
| 907 |
break; |
| 908 |
} |
| 909 |
|
| 910 |
for (int iproc = Proc_Info[4]; iproc < Proc_Info[4] + Proc_Info[5]; iproc++) { |
| 911 |
size_t itotal_nodes = globals.Num_Internal_Nodes[iproc] + globals.Num_Border_Nodes[iproc] + |
| 912 |
globals.Num_External_Nodes[iproc]; |
| 913 |
|
| 914 |
for (size_t j = 0; j < itotal_nodes; j++) { |
| 915 |
size_t inode = globals.GNodes[iproc][j]; |
| 916 |
globals.Coor[iproc][idim][j] = coord[inode]; |
| 917 |
} |
| 918 |
} |
| 919 |
} |
| 920 |
coord.clear(); |
| 921 |
|
| 922 |
for (int i = 0; i < globals.Num_Dim; i++) { |
| 923 |
Coord_Name[i] = reinterpret_cast<char *>( |
| 924 |
array_alloc(__FILE__, __LINE__, 1, max_name_length + 1, sizeof(char))); |
| 925 |
} |
| 926 |
|
| 927 |
/* Get the coordinate names */ |
| 928 |
if (ex_get_coord_names(exoid, Coord_Name) < 0) { |
| 929 |
fmt::print(stderr, "ERROR:Unable to obtain coordinate names\n"); |
| 930 |
exit(1); |
| 931 |
} |
| 932 |
|
| 933 |
/* Handle global node ids... */ |
| 934 |
std::vector<INT> global_node_ids(globals.Num_Node); |
| 935 |
|
| 936 |
check_exodus_error(ex_get_id_map(exoid, EX_NODE_MAP, Data(global_node_ids)), "ex_get_id_map"); |
| 937 |
/* |
| 938 |
* Check whether map is sequential (1..globals.Num_Node). If it is, then it |
| 939 |
* provides no information and we don't need to store it in the |
| 940 |
* output databases. |
| 941 |
*/ |
| 942 |
{ |
| 943 |
bool sequential = true; |
| 944 |
for (size_t i = 0; i < globals.Num_Node; i++) { |
| 945 |
if ((size_t)global_node_ids[i] != i + 1) { |
| 946 |
sequential = false; |
| 947 |
break; |
| 948 |
} |
| 949 |
} |
| 950 |
|
| 951 |
// Check that map is valid 1 <= global_node_id[*] |
| 952 |
// If not, output a warning and disable the map. |
| 953 |
for (size_t i = 0; i < globals.Num_Node; i++) { |
| 954 |
if (global_node_ids[i] <= 0) { |
| 955 |
fmt::print(stderr, |
| 956 |
"---------------------------------------------------------------------\n" |
| 957 |
"ERROR: Local node {} has a global id of {} which is invalid.\n" |
| 958 |
" All global ids must be greater than 0. The map will be ignored.\n" |
| 959 |
"---------------------------------------------------------------------\n", |
| 960 |
fmt::group_digits(i + 1), fmt::group_digits(global_node_ids[i])); |
| 961 |
sequential = true; // Map is invalid, ignore it. |
| 962 |
break; |
| 963 |
} |
| 964 |
} |
| 965 |
|
| 966 |
if (!sequential) { |
| 967 |
for (int iproc = Proc_Info[4]; iproc < Proc_Info[4] + Proc_Info[5]; iproc++) { |
| 968 |
|
| 969 |
size_t itotal_nodes = globals.Num_Internal_Nodes[iproc] + globals.Num_Border_Nodes[iproc] + |
| 970 |
globals.Num_External_Nodes[iproc]; |
| 971 |
|
| 972 |
globals.Proc_Global_Node_Id_Map[iproc].resize(itotal_nodes); |
| 973 |
extract_global_node_ids(global_node_ids, globals.Num_Node, iproc); |
| 974 |
} |
| 975 |
} |
| 976 |
} |
| 977 |
} /* END of routine read_coord */ |
| 978 |
|
| 979 |
/*****************************************************************************/ |
| 980 |
/*****************************************************************************/ |
| 981 |
/*****************************************************************************/ |
| 982 |
|
| 983 |
template <typename T, typename INT> void NemSpread<T, INT>::extract_elem_blk() |
| 984 |
|
| 985 |
/* Function which calculates the element block information for the current |
| 986 |
* processor, given the global element block information. If an error is |
| 987 |
* tripped, the program will set a flag, and abort at the end of the routine. |
| 988 |
* |
| 989 |
* The following global variables are allocated and calculated in this routine: |
| 990 |
* |
| 991 |
* globals.Num_Elem_Blk = Number of element blocks on the current |
| 992 |
* processor |
| 993 |
* globals.GElem_Blks [globals.Num_Elem_Blk] |
| 994 |
* = Map from the local element block number |
| 995 |
* to the global element block number. |
| 996 |
* globals.Proc_Nodes_Per_Elem [globals.Num_Elem_Blk] |
| 997 |
* = Number of nodes per element for each |
| 998 |
* block on the current processor |
| 999 |
* globals.Proc_Elem_Blk_Ids [globals.Num_Elem_Blk] |
| 1000 |
* = Element block id's for the processor's |
| 1001 |
* element blocks |
| 1002 |
* globals.Proc_Elem_Blk_Types [globals.Num_Elem_Blk] |
| 1003 |
* = Element block types for the processor's |
| 1004 |
* element blocks |
| 1005 |
* (this is a unique integer number) |
| 1006 |
* globals.Proc_Num_Attr [globals.Num_Elem_Blk] |
| 1007 |
* = Number of attributes for each block on the |
| 1008 |
* current processor |
| 1009 |
* globals.Proc_Num_Elem_In_Blk [globals.Num_Elem_Blk] |
| 1010 |
* = Number of elements in the processor's |
| 1011 |
* element blocks |
| 1012 |
* |
| 1013 |
* ------------------------------------------------------------------------ |
| 1014 |
* |
| 1015 |
* Functions called: |
| 1016 |
* |
| 1017 |
* find_elem_block -- function which finds the element block which |
| 1018 |
* owns each element in this processor. In addition, |
| 1019 |
* it determines a map from the current element to an |
| 1020 |
* element block. |
| 1021 |
* |
| 1022 |
* ------------------------------------------------------------------------ |
| 1023 |
*/ |
| 1024 |
|
| 1025 |
{ |
| 1026 |
std::vector<INT> proc_elem_blk{}; |
| 1027 |
|
| 1028 |
/* Element blocks local to the current processor */ |
| 1029 |
|
| 1030 |
/**************************** execution begins ******************************/ |
| 1031 |
|
| 1032 |
/* |
| 1033 |
* Allocate temporary array for a mapping between the local element number |
| 1034 |
* and the corresponding element block id for the element block that the |
| 1035 |
* element belongs to. |
| 1036 |
*/ |
| 1037 |
|
| 1038 |
for (int iproc = Proc_Info[4]; iproc < Proc_Info[4] + Proc_Info[5]; iproc++) { |
| 1039 |
|
| 1040 |
proc_elem_blk.resize(globals.Num_Internal_Elems[iproc] + globals.Num_Border_Elems[iproc]); |
| 1041 |
|
| 1042 |
/* Find out which element block each element in this processor belongs to. |
| 1043 |
* Fill this information into the temporary vector, proc_elem_blk. Also, |
| 1044 |
* calculate: |
| 1045 |
* |
| 1046 |
* globals.Num_Elem_Blk = Number of element blocks defined on |
| 1047 |
* the current processor. |
| 1048 |
* globals.GElem_Blks = Map from the local element block number |
| 1049 |
* to the global element block number. |
| 1050 |
*/ |
| 1051 |
find_elem_block(proc_elem_blk, iproc, Proc_Ids[iproc]); |
| 1052 |
|
| 1053 |
/* Allocate Permanent integer arrays, which have lengths equal to the |
| 1054 |
* number of element blocks defined on the processor. This is done with a |
| 1055 |
* single malloc call. Initialize this space to zero. |
| 1056 |
*/ |
| 1057 |
if (globals.Num_Elem_Blk > 0) { |
| 1058 |
globals.Proc_Nodes_Per_Elem[iproc] = (INT *)array_alloc( |
| 1059 |
__FILE__, __LINE__, 1, (4 * globals.Num_Elem_Blk + globals.Proc_Num_Elem_Blk[iproc]), |
| 1060 |
sizeof(INT)); |
| 1061 |
globals.Proc_Elem_Blk_Ids[iproc] = globals.Proc_Nodes_Per_Elem[iproc] + globals.Num_Elem_Blk; |
| 1062 |
globals.Proc_Elem_Blk_Types[iproc] = globals.Proc_Elem_Blk_Ids[iproc] + globals.Num_Elem_Blk; |
| 1063 |
globals.Proc_Num_Attr[iproc] = |
| 1064 |
globals.Proc_Elem_Blk_Types[iproc] + globals.Proc_Num_Elem_Blk[iproc]; |
| 1065 |
globals.Proc_Num_Elem_In_Blk[iproc] = globals.Proc_Num_Attr[iproc] + globals.Num_Elem_Blk; |
| 1066 |
/* Initialize */ |
| 1067 |
for (int i = 0; i < (4 * globals.Num_Elem_Blk + globals.Proc_Num_Elem_Blk[iproc]); i++) { |
| 1068 |
globals.Proc_Nodes_Per_Elem[iproc][i] = 0; |
| 1069 |
} |
| 1070 |
} |
| 1071 |
else { |
| 1072 |
fmt::print(stderr, "ERROR globals.Num_Elem_Blk = {}\n", globals.Num_Elem_Blk); |
| 1073 |
exit(1); |
| 1074 |
} |
| 1075 |
|
| 1076 |
/* |
| 1077 |
* Fill in the local processor element block arrays from a lookup from the |
| 1078 |
* global element block arrays. Note that the global element block arrays |
| 1079 |
* will be freed at the end of the file. |
| 1080 |
*/ |
| 1081 |
for (int i = 0; i < globals.Proc_Num_Elem_Blk[iproc]; i++) { |
| 1082 |
|
| 1083 |
size_t iglobal_blk = globals.GElem_Blks[iproc][i]; |
| 1084 |
globals.Proc_Nodes_Per_Elem[iproc][i] = Num_Nodes_Per_Elem[iglobal_blk]; |
| 1085 |
globals.Proc_Elem_Blk_Ids[iproc][i] = Elem_Blk_Ids[iglobal_blk]; |
| 1086 |
globals.Proc_Num_Attr[iproc][i] = Num_Attr_Per_Elem[iglobal_blk]; |
| 1087 |
|
| 1088 |
/* Determine the element type integer id for this current element block. |
| 1089 |
* The element type integer ID is a unique identifier (unique for all |
| 1090 |
* element types that the program knows about). It is determined from |
| 1091 |
* the character string name, Elem_Blk_Types, and the number of nodes in |
| 1092 |
* the element. |
| 1093 |
*/ |
| 1094 |
globals.Proc_Elem_Blk_Types[iproc][i] = get_type( |
| 1095 |
Elem_Blk_Types[iglobal_blk], globals.Proc_Nodes_Per_Elem[iproc][i], globals.Num_Dim); |
| 1096 |
} |
| 1097 |
|
| 1098 |
/* |
| 1099 |
* Determine the number of elements in each of the element blocks defined |
| 1100 |
* on the local processor. |
| 1101 |
*/ |
| 1102 |
|
| 1103 |
for (int i = 0; i < globals.Proc_Num_Elem_Blk[iproc]; i++) { |
| 1104 |
for (INT j = 0; j < globals.Num_Internal_Elems[iproc] + globals.Num_Border_Elems[iproc]; |
| 1105 |
j++) { |
| 1106 |
if (proc_elem_blk[j] == globals.Proc_Elem_Blk_Ids[iproc][i]) { |
| 1107 |
(globals.Proc_Num_Elem_In_Blk[iproc][i])++; |
| 1108 |
} |
| 1109 |
} |
| 1110 |
} |
| 1111 |
|
| 1112 |
/* Sort globals.GElems so that each element block is monotonic */ |
| 1113 |
size_t j = 0; |
| 1114 |
for (int i = 0; i < globals.Proc_Num_Elem_Blk[iproc]; i++) { |
| 1115 |
gds_qsort((Data(globals.GElems[iproc])) + j, globals.Proc_Num_Elem_In_Blk[iproc][i]); |
| 1116 |
j += globals.Proc_Num_Elem_In_Blk[iproc][i]; |
| 1117 |
} |
| 1118 |
} /* End "for(iproc=0; iproc <Proc_Info[2]; iproc++)" */ |
| 1119 |
|
| 1120 |
if (Debug_Flag >= 5) { |
| 1121 |
for (int iproc = Proc_Info[4]; iproc < Proc_Info[4] + Proc_Info[5]; iproc++) { |
| 1122 |
|
| 1123 |
/* Printout the Element Block Information defined on each Processor */ |
| 1124 |
print_line("=", 79); |
| 1125 |
fmt::print("\t\tLocal Element Block information for Proc = {}\n", Proc_Ids[iproc]); |
| 1126 |
fmt::print("\t\tNumber of Elem blocks on processor = {}\n", globals.Proc_Num_Elem_Blk[iproc]); |
| 1127 |
fmt::print("{}{}\n", "Local_Block_Num Global_Block_Num Block_ID Nodes_Per_Elem ", |
| 1128 |
"Num_Attributes Elem_Blk_Type globals.Proc_Num_Elem_In_Blk " |
| 1129 |
"Glb_Elm_In_Blk"); |
| 1130 |
print_line("-", 79); |
| 1131 |
for (int i = 0; i < globals.Proc_Num_Elem_Blk[iproc]; i++) { |
| 1132 |
fmt::print("{:4d}\t\t{:5}\t{:8}\t{:8}\t{:8}\t{:8}\t{:8}\t{:8}\n", i, |
| 1133 |
globals.GElem_Blks[iproc][i], |
| 1134 |
fmt::group_digits(globals.Proc_Elem_Blk_Ids[iproc][i]), |
| 1135 |
fmt::group_digits(globals.Proc_Nodes_Per_Elem[iproc][i]), |
| 1136 |
fmt::group_digits(globals.Proc_Num_Attr[iproc][i]), |
| 1137 |
globals.Proc_Elem_Blk_Types[iproc][i], |
| 1138 |
fmt::group_digits(globals.Proc_Num_Elem_In_Blk[iproc][i]), |
| 1139 |
fmt::group_digits(Num_Elem_In_Blk[globals.GElem_Blks[iproc][i]])); |
| 1140 |
} |
| 1141 |
print_line("=", 79); |
| 1142 |
} |
| 1143 |
} |
| 1144 |
|
| 1145 |
} /* END extract_elem_blk() */ |
| 1146 |
|
| 1147 |
/*****************************************************************************/ |
| 1148 |
/*****************************************************************************/ |
| 1149 |
/*****************************************************************************/ |
| 1150 |
|
| 1151 |
template <typename T, typename INT> void NemSpread<T, INT>::read_elem_blk(int exoid) |
| 1152 |
|
| 1153 |
/* Function which reads the element block information from an EXODUS |
| 1154 |
* database. If an error is tripped, the program will set a flag, and abort at |
| 1155 |
* the end of the routine. |
| 1156 |
* |
| 1157 |
* ------------------------------------------------------------------------ |
| 1158 |
* |
| 1159 |
* extract_elem_connect |
| 1160 |
* - Function which sorts through the element block |
| 1161 |
* connectivity information and extracts the |
| 1162 |
* information required by each processor. |
| 1163 |
* |
| 1164 |
* ------------------------------------------------------------------------ |
| 1165 |
*/ |
| 1166 |
|
| 1167 |
{ |
| 1168 |
|
| 1169 |
/* Local variables */ |
| 1170 |
|
| 1171 |
T *elem_attr = nullptr; |
| 1172 |
#ifdef DEBUG |
| 1173 |
size_t ielem_count; |
| 1174 |
#endif |
| 1175 |
INT *elem_blk = nullptr, ipos; |
| 1176 |
size_t num_elem_per_message = 0; |
| 1177 |
size_t num_attr_per_message = 0; |
| 1178 |
size_t num_attr_left_over = 0; |
| 1179 |
size_t num_elem_messages = 0; |
| 1180 |
size_t num_attr_messages = 0; |
| 1181 |
size_t num_elem_left_over = 0; |
| 1182 |
|
| 1183 |
size_t istart_elem; |
| 1184 |
size_t iend_elem; |
| 1185 |
size_t istart_attr; |
| 1186 |
size_t iend_attr; |
| 1187 |
int local_ielem_blk; |
| 1188 |
|
| 1189 |
/**************************** execution begins ******************************/ |
| 1190 |
|
| 1191 |
/* |
| 1192 |
* Allocate memory for the vector of global element types in the entire |
| 1193 |
* mesh. Used in sideset manipulations |
| 1194 |
*/ |
| 1195 |
GM_Elem_Types = (int *)array_alloc(__FILE__, __LINE__, 1, globals.Num_Elem, sizeof(int)); |
| 1196 |
|
| 1197 |
std::vector<INT> global_ids(globals.Num_Elem); |
| 1198 |
|
| 1199 |
for (int iproc = Proc_Info[4]; iproc < Proc_Info[4] + Proc_Info[5]; iproc++) { |
| 1200 |
|
| 1201 |
/* |
| 1202 |
* Build globals.Proc_Connect_Ptr, a vector of pointers to the start of each |
| 1203 |
* element's connectivity list in globals.Proc_Elem_Connect. The last entry in |
| 1204 |
* globals.Proc_Connect_Ptr is the length of globals.Proc_Elem_Connect. |
| 1205 |
*/ |
| 1206 |
globals.Proc_Connect_Ptr[iproc] = (INT *)array_alloc( |
| 1207 |
__FILE__, __LINE__, 1, |
| 1208 |
globals.Num_Internal_Elems[iproc] + globals.Num_Border_Elems[iproc] + 1, sizeof(INT)); |
| 1209 |
|
| 1210 |
size_t iptr_count = 0; |
| 1211 |
size_t iconnect_length = 0; |
| 1212 |
for (int i = 0; i < globals.Proc_Num_Elem_Blk[iproc]; i++) { |
| 1213 |
for (INT j = 0; j < globals.Proc_Num_Elem_In_Blk[iproc][i]; j++) { |
| 1214 |
globals.Proc_Connect_Ptr[iproc][iptr_count++] = iconnect_length; |
| 1215 |
iconnect_length += globals.Proc_Nodes_Per_Elem[iproc][i]; |
| 1216 |
} |
| 1217 |
} |
| 1218 |
globals.Proc_Connect_Ptr[iproc][iptr_count] = iconnect_length; |
| 1219 |
|
| 1220 |
/* |
| 1221 |
* Allocate the processor's connectivity list vector |
| 1222 |
* - This is a global vector |
| 1223 |
*/ |
| 1224 |
globals.Proc_Elem_Connect[iproc] = |
| 1225 |
(INT *)array_alloc(__FILE__, __LINE__, 1, iconnect_length, sizeof(INT)); |
| 1226 |
|
| 1227 |
#ifdef DEBUG |
| 1228 |
for (size_t i = 0; i < iconnect_length; i++) |
| 1229 |
globals.Proc_Elem_Connect[iproc][i] = -1111111; |
| 1230 |
#endif |
| 1231 |
|
| 1232 |
/* |
| 1233 |
* Allocate the processor's attribute list vector, if its length |
| 1234 |
* is nonzero |
| 1235 |
*/ |
| 1236 |
size_t iattr_length = 0; |
| 1237 |
for (int i = 0; i < globals.Proc_Num_Elem_Blk[iproc]; i++) { |
| 1238 |
iattr_length += globals.Proc_Num_Attr[iproc][i] * globals.Proc_Num_Elem_In_Blk[iproc][i]; |
| 1239 |
} |
| 1240 |
if (iattr_length > 0) { |
| 1241 |
globals.Proc_Elem_Attr[iproc].resize(iattr_length); |
| 1242 |
} |
| 1243 |
} /* End "for(iproc=0; iproc <Proc_Info[2]; iproc)" */ |
| 1244 |
|
| 1245 |
size_t icount = 0; |
| 1246 |
|
| 1247 |
/* |
| 1248 |
* READ THE ELEMENT CONNECTIVITY AND ATTRIBUTE INFORMATION |
| 1249 |
* FROM THE EXODUS FILE FOR EACH ELEMENT BLOCK |
| 1250 |
*/ |
| 1251 |
for (int ielem_blk = 0; ielem_blk < globals.Num_Elem_Blk; ielem_blk++) { |
| 1252 |
if (Num_Elem_In_Blk[ielem_blk] > 0) { |
| 1253 |
|
| 1254 |
/* |
| 1255 |
* Calculate the size of a single element's connectivity list and a |
| 1256 |
* single element's attribute list |
| 1257 |
*/ |
| 1258 |
size_t iconnect_size = Num_Nodes_Per_Elem[ielem_blk] * sizeof(INT); |
| 1259 |
size_t iattr_size = Num_Attr_Per_Elem[ielem_blk] * sizeof(T); |
| 1260 |
|
| 1261 |
/* |
| 1262 |
* Determine how many element connectivity list and how many element |
| 1263 |
* attributes can be send in single message |
| 1264 |
*/ |
| 1265 |
find_message_info(iconnect_size, Num_Elem_In_Blk[ielem_blk], &num_elem_per_message, |
| 1266 |
&num_elem_messages, &num_elem_left_over); |
| 1267 |
|
| 1268 |
if (iattr_size > 0) { |
| 1269 |
find_message_info(iattr_size, Num_Elem_In_Blk[ielem_blk], &num_attr_per_message, |
| 1270 |
&num_attr_messages, &num_attr_left_over); |
| 1271 |
} |
| 1272 |
else { |
| 1273 |
num_attr_messages = 0; |
| 1274 |
} |
| 1275 |
|
| 1276 |
if (Debug_Flag > 1) { |
| 1277 |
fmt::print("\n\nMessage summary for Element Block number {}, ", ielem_blk); |
| 1278 |
fmt::print("having a block id of {}:\n", Elem_Blk_Ids[ielem_blk]); |
| 1279 |
fmt::print("\tNumber of messages needed for the element connectivity " |
| 1280 |
"vector = {}\n", |
| 1281 |
num_elem_messages); |
| 1282 |
fmt::print("\tNumber of elements per message = {}\n", num_elem_per_message); |
| 1283 |
fmt::print("\tNumber of nodes per element = {}\n", Num_Nodes_Per_Elem[ielem_blk]); |
| 1284 |
fmt::print("\tLength of each message = {} bytes\n", |
| 1285 |
(Num_Nodes_Per_Elem[ielem_blk] * num_elem_per_message * sizeof(INT))); |
| 1286 |
if (num_attr_messages > 0) { |
| 1287 |
fmt::print("\tNumber of attribute messages: {}\n\tNumber " |
| 1288 |
"of attributes per message: {}\n\n", |
| 1289 |
num_attr_messages, num_attr_per_message); |
| 1290 |
} |
| 1291 |
} |
| 1292 |
|
| 1293 |
/* |
| 1294 |
* Allocate the arrays for reading from the ExodusII file and |
| 1295 |
* broadcasting |
| 1296 |
*/ |
| 1297 |
elem_blk = (INT *)array_alloc( |
| 1298 |
__FILE__, __LINE__, 1, Num_Nodes_Per_Elem[ielem_blk] * num_elem_per_message, sizeof(INT)); |
| 1299 |
if (num_attr_messages > 0) { |
| 1300 |
elem_attr = (T *)array_alloc( |
| 1301 |
__FILE__, __LINE__, 1, Num_Attr_Per_Elem[ielem_blk] * num_attr_per_message, sizeof(T)); |
| 1302 |
} |
| 1303 |
else { |
| 1304 |
elem_attr = nullptr; |
| 1305 |
} |
| 1306 |
|
| 1307 |
/* |
| 1308 |
* Read in the element connectivity list for the current element block, |
| 1309 |
* ielem_blk, from Proc 0, and then, broadcast it to all of the processors |
| 1310 |
*/ |
| 1311 |
for (size_t i = 0; i < num_elem_messages; i++) { |
| 1312 |
|
| 1313 |
if (Debug_Flag >= 2) { |
| 1314 |
fmt::print("\telem block message: {} of {}\n", i + 1, num_elem_messages); |
| 1315 |
} |
| 1316 |
|
| 1317 |
/* Initialize the element connectivity list to a value of -1.0 */ |
| 1318 |
for (size_t j = 0; j < Num_Nodes_Per_Elem[ielem_blk] * num_elem_per_message; |
| 1319 |
elem_blk[j++] = -1) { |
| 1320 |
; |
| 1321 |
} |
| 1322 |
|
| 1323 |
size_t num_to_get = 0; |
| 1324 |
istart_elem = i * num_elem_per_message; |
| 1325 |
if (num_elem_left_over == 0 || i < num_elem_messages - 1) { |
| 1326 |
num_to_get = num_elem_per_message; |
| 1327 |
} |
| 1328 |
else { |
| 1329 |
num_to_get = num_elem_left_over; |
| 1330 |
} |
| 1331 |
iend_elem = istart_elem + num_to_get; |
| 1332 |
|
| 1333 |
int el_type; |
| 1334 |
check_exodus_error(ex_get_partial_conn(exoid, EX_ELEM_BLOCK, Elem_Blk_Ids[ielem_blk], |
| 1335 |
(istart_elem + 1), num_to_get, elem_blk, nullptr, |
| 1336 |
nullptr), |
| 1337 |
"ex_get_partial_conn"); |
| 1338 |
if (Debug_Flag >= 2) { |
| 1339 |
fmt::print("\t\tread connectivity\n"); |
| 1340 |
} |
| 1341 |
|
| 1342 |
el_type = |
| 1343 |
get_type(Elem_Blk_Types[ielem_blk], Num_Nodes_Per_Elem[ielem_blk], globals.Num_Dim); |
| 1344 |
for (size_t ielem = 0; ielem < num_to_get; ielem++) { |
| 1345 |
GM_Elem_Types[icount++] = el_type; |
| 1346 |
} |
| 1347 |
|
| 1348 |
if (Debug_Flag >= 2) { |
| 1349 |
fmt::print("\t\tgot element types\n"); |
| 1350 |
} |
| 1351 |
|
| 1352 |
/* PRINT OUT THE ELEMENT CONNECTIVITY TABLE IF IN DEBUGGING MODE */ |
| 1353 |
if (Debug_Flag >= 6) { |
| 1354 |
fmt::print("\n\n\n"); |
| 1355 |
print_line("=", 79); |
| 1356 |
fmt::print("Printout of Element connectivity list obtained from " |
| 1357 |
"Exodus file:\n"); |
| 1358 |
fmt::print("\tGlobal element block number = {}\n", ielem_blk); |
| 1359 |
fmt::print("\tElement ID number = {}\n", Elem_Blk_Ids[ielem_blk]); |
| 1360 |
fmt::print("\tMessage number = {}\n", i); |
| 1361 |
print_line("-", 79); |
| 1362 |
ipos = 0; |
| 1363 |
for (size_t j = 0; j < num_to_get; j++) { |
| 1364 |
fmt::print("\t elem: {}, nodes:", j); |
| 1365 |
for (int k = 0; k < Num_Nodes_Per_Elem[ielem_blk]; k++) { |
| 1366 |
fmt::print(" {}", elem_blk[ipos++]); |
| 1367 |
} |
| 1368 |
fmt::print("\n"); |
| 1369 |
} |
| 1370 |
print_line("=", 79); |
| 1371 |
} |
| 1372 |
|
| 1373 |
/* |
| 1374 |
* On each processor, extract the element connectivity lists that the |
| 1375 |
* processor needs |
| 1376 |
*/ |
| 1377 |
for (int iproc = Proc_Info[4]; iproc < Proc_Info[4] + Proc_Info[5]; iproc++) { |
| 1378 |
extract_elem_connect(elem_blk, ielem_blk, istart_elem, iend_elem, &local_ielem_blk, |
| 1379 |
iproc); |
| 1380 |
} |
| 1381 |
if (Debug_Flag >= 2) { |
| 1382 |
fmt::print("\t\textract connectivity\n"); |
| 1383 |
} |
| 1384 |
|
| 1385 |
} /* End "for(i=0; i < num_elem_messages; i++)" */ |
| 1386 |
|
| 1387 |
/* Read in the element attribute lists and broadcast to the processors */ |
| 1388 |
for (size_t i = 0; i < num_attr_messages; i++) { |
| 1389 |
|
| 1390 |
if (Debug_Flag >= 2) { |
| 1391 |
fmt::print("\tattribute message: {} of {}\n", i + 1, num_attr_messages); |
| 1392 |
} |
| 1393 |
|
| 1394 |
/* Initialize */ |
| 1395 |
for (size_t j = 0; j < Num_Attr_Per_Elem[ielem_blk] * num_attr_per_message; j++) { |
| 1396 |
elem_attr[j] = 0.0; |
| 1397 |
} |
| 1398 |
|
| 1399 |
istart_attr = i * num_attr_per_message; |
| 1400 |
|
| 1401 |
if (num_attr_left_over == 0 || i < (num_attr_messages - 1)) { |
| 1402 |
iend_attr = istart_attr + num_attr_per_message; |
| 1403 |
} |
| 1404 |
else { |
| 1405 |
iend_attr = istart_attr + num_attr_left_over; |
| 1406 |
} |
| 1407 |
|
| 1408 |
if (num_attr_left_over == 0 || i < (num_attr_messages - 1)) { |
| 1409 |
check_exodus_error(ex_get_partial_attr(exoid, EX_ELEM_BLOCK, Elem_Blk_Ids[ielem_blk], |
| 1410 |
(istart_attr + 1), num_attr_per_message, |
| 1411 |
elem_attr), |
| 1412 |
"ex_get_partial_attr"); |
| 1413 |
} |
| 1414 |
else { |
| 1415 |
check_exodus_error(ex_get_partial_attr(exoid, EX_ELEM_BLOCK, Elem_Blk_Ids[ielem_blk], |
| 1416 |
(istart_attr + 1), num_attr_left_over, elem_attr), |
| 1417 |
"ex_get_partial_attr"); |
| 1418 |
} |
| 1419 |
|
| 1420 |
for (int iproc = Proc_Info[4]; iproc < Proc_Info[4] + Proc_Info[5]; iproc++) { |
| 1421 |
|
| 1422 |
if (Debug_Flag > 6) { |
| 1423 |
fmt::print("\t\tExtract attributes for processor {}\n", Proc_Ids[iproc]); |
| 1424 |
} |
| 1425 |
/* |
| 1426 |
* On each processor, extract the element attributes that the |
| 1427 |
* processor needs |
| 1428 |
*/ |
| 1429 |
extract_elem_attr(elem_attr, ielem_blk, istart_attr, iend_attr, |
| 1430 |
Num_Attr_Per_Elem[ielem_blk], iproc); |
| 1431 |
} |
| 1432 |
} |
| 1433 |
|
| 1434 |
/* Free vectors */ |
| 1435 |
safe_free((void **)&elem_blk); |
| 1436 |
safe_free((void **)&elem_attr); |
| 1437 |
|
| 1438 |
} /* END "if (globals.Proc_Num_Elem_In_Blk[ielem_blk] > 0)" */ |
| 1439 |
|
| 1440 |
} /* End "for(ielem_blk=0; ielem_blk < globals.Num_Elem_Blk; ielem_blk++)" */ |
| 1441 |
|
| 1442 |
/* Handle global element ids... */ |
| 1443 |
check_exodus_error(ex_get_id_map(exoid, EX_ELEM_MAP, Data(global_ids)), "ex_get_id_map"); |
| 1444 |
|
| 1445 |
/* |
| 1446 |
* Check whether map is sequential (1..globals.Num_Elem). If it is, then it |
| 1447 |
* provides no information and we don't need to store it in the |
| 1448 |
* output databases. |
| 1449 |
*/ |
| 1450 |
{ |
| 1451 |
bool sequential = true; |
| 1452 |
for (size_t i = 0; i < globals.Num_Elem; i++) { |
| 1453 |
if ((size_t)global_ids[i] != i + 1) { |
| 1454 |
sequential = false; |
| 1455 |
break; |
| 1456 |
} |
| 1457 |
} |
| 1458 |
|
| 1459 |
// Check that map is valid 1 <= global_node_id[*] |
| 1460 |
// If not, output a warning and disable the map. |
| 1461 |
for (size_t i = 0; i < globals.Num_Elem; i++) { |
| 1462 |
if (global_ids[i] <= 0) { |
| 1463 |
fmt::print(stderr, |
| 1464 |
"---------------------------------------------------------------------\n" |
| 1465 |
"ERROR: Local element {} has a global id of {} which is invalid.\n" |
| 1466 |
" All global ids must be greater than 0. The map will be ignored.\n" |
| 1467 |
"---------------------------------------------------------------------\n", |
| 1468 |
fmt::group_digits(i + 1), fmt::group_digits(global_ids[i])); |
| 1469 |
sequential = true; // Map is invalid, ignore it. |
| 1470 |
break; |
| 1471 |
} |
| 1472 |
} |
| 1473 |
|
| 1474 |
if (!sequential) { |
| 1475 |
for (int iproc = Proc_Info[4]; iproc < Proc_Info[4] + Proc_Info[5]; iproc++) { |
| 1476 |
globals.Proc_Global_Elem_Id_Map[iproc].resize(globals.Num_Internal_Elems[iproc] + |
| 1477 |
globals.Num_Border_Elems[iproc]); |
| 1478 |
extract_global_element_ids(global_ids, globals.Num_Elem, iproc); |
| 1479 |
} |
| 1480 |
} |
| 1481 |
else { |
| 1482 |
/* Should be nullptr already, but make it more clear */ |
| 1483 |
for (int iproc = Proc_Info[4]; iproc < Proc_Info[4] + Proc_Info[5]; iproc++) { |
| 1484 |
globals.Proc_Global_Elem_Id_Map[iproc].clear(); |
| 1485 |
} |
| 1486 |
} |
| 1487 |
} |
| 1488 |
} /* read_elem_blk */ |
| 1489 |
|
| 1490 |
/*****************************************************************************/ |
| 1491 |
/*****************************************************************************/ |
| 1492 |
/*****************************************************************************/ |
| 1493 |
|
| 1494 |
template <typename T, typename INT> |
| 1495 |
void NemSpread<T, INT>::extract_global_element_ids(const std::vector<INT> &global_ids, |
| 1496 |
size_t /*Num_Elem*/, int iproc) |
| 1497 |
{ |
| 1498 |
/* |
| 1499 |
* globals.Num_Elem -- number of elements in serial mesh. |
| 1500 |
* global_ids[] (size globals.Num_Elem) = global id corresponding to index |
| 1501 |
* of element in serial file |
| 1502 |
* globals.Num_Internal_Elems[iproc] + globals.Num_Border_Elems[iproc] = number of element on this |
| 1503 |
processor |
| 1504 |
* globals.GElems[iproc][i] = index of element in serial file corresponding |
| 1505 |
to i'th element on this processor. Both are zero-based? |
| 1506 |
*/ |
| 1507 |
size_t num_local_element = globals.Num_Internal_Elems[iproc] + globals.Num_Border_Elems[iproc]; |
| 1508 |
for (size_t i = 0; i < num_local_element; i++) { |
| 1509 |
globals.Proc_Global_Elem_Id_Map[iproc][i] = global_ids[globals.GElems[iproc][i]]; |
| 1510 |
} |
| 1511 |
} |
| 1512 |
|
| 1513 |
/*****************************************************************************/ |
| 1514 |
/*****************************************************************************/ |
| 1515 |
/*****************************************************************************/ |
| 1516 |
|
| 1517 |
template <typename T, typename INT> |
| 1518 |
void NemSpread<T, INT>::extract_global_node_ids(const std::vector<INT> &global_ids, |
| 1519 |
size_t /*Num_Node*/, int iproc) |
| 1520 |
{ |
| 1521 |
/* |
| 1522 |
* Num_Elem -- number of elements in serial mesh. |
| 1523 |
* global_ids[] (size Num_Elem) = global id corresponding to index |
| 1524 |
* of element in serial file |
| 1525 |
* Num_Internal_Elems[iproc] + Num_Border_Elems[iproc] = number of element on this processor |
| 1526 |
* GElems[iproc][i] = index of element in serial file corresponding |
| 1527 |
* to i'th element on this processor. Both are zero-based? |
| 1528 |
*/ |
| 1529 |
size_t num_local_node = globals.Num_Internal_Nodes[iproc] + globals.Num_Border_Nodes[iproc] + |
| 1530 |
globals.Num_External_Nodes[iproc]; |
| 1531 |
for (size_t i = 0; i < num_local_node; i++) { |
| 1532 |
globals.Proc_Global_Node_Id_Map[iproc][i] = global_ids[globals.GNodes[iproc][i]]; |
| 1533 |
} |
| 1534 |
} |
| 1535 |
|
| 1536 |
/*****************************************************************************/ |
| 1537 |
/*****************************************************************************/ |
| 1538 |
/*****************************************************************************/ |
| 1539 |
|
| 1540 |
template <typename T, typename INT> |
| 1541 |
size_t NemSpread<T, INT>::extract_elem_connect(INT elem_blk[], int icurrent_elem_blk, |
| 1542 |
size_t istart_elem, size_t iend_elem, |
| 1543 |
int *local_ielem_blk, int iproc) |
| 1544 |
|
| 1545 |
/* |
| 1546 |
* Function which extracts the element connectivity list from the current |
| 1547 |
* element connectivity message. The results are put into the global |
| 1548 |
* vector, globals.Proc_Elem_Connect. |
| 1549 |
* |
| 1550 |
* The function returns the number of elements in the current message |
| 1551 |
* that are defined on the current processor. |
| 1552 |
*/ |
| 1553 |
|
| 1554 |
{ |
| 1555 |
size_t count_hits = 0; |
| 1556 |
bool found = false; |
| 1557 |
|
| 1558 |
/* Match the Element Block Id of the global block with the local block number |
| 1559 |
* The end result is the current value of ielem_blk, Store this value in the |
| 1560 |
* output variable, local_ielem_blk. |
| 1561 |
*/ |
| 1562 |
for (int ielem_blk = 0; ielem_blk < globals.Proc_Num_Elem_Blk[iproc]; ielem_blk++) { |
| 1563 |
if (globals.Proc_Elem_Blk_Ids[iproc][ielem_blk] == Elem_Blk_Ids[icurrent_elem_blk]) { |
| 1564 |
*local_ielem_blk = ielem_blk; |
| 1565 |
found = true; |
| 1566 |
|
| 1567 |
/* Some elements in the current element block passed in from |
| 1568 |
read_elem_blk may be on this processor */ |
| 1569 |
|
| 1570 |
/* Calculate the number of elements in the current message */ |
| 1571 |
size_t num_elem = iend_elem - istart_elem; |
| 1572 |
|
| 1573 |
/* Calculate iglobal_offset - The sum of the elements in all the global |
| 1574 |
* element blocks preceding this one |
| 1575 |
*/ |
| 1576 |
size_t iglobal_offset = 0; |
| 1577 |
for (int i = 0; i < globals.GElem_Blks[iproc][ielem_blk]; i++) { |
| 1578 |
iglobal_offset += Num_Elem_In_Blk[i]; |
| 1579 |
} |
| 1580 |
|
| 1581 |
/* Calculate iproc_offset - The sum of the elements in all the |
| 1582 |
* processor's |
| 1583 |
* element blocks preceding this one. |
| 1584 |
* Calculate iproc_start - The starting position in the connectivity |
| 1585 |
* vector for this element block, |
| 1586 |
* globals.Proc_Elem_Connect[] |
| 1587 |
*/ |
| 1588 |
size_t iproc_offset = 0; |
| 1589 |
size_t iproc_start = 0; |
| 1590 |
for (int i = 0; i < ielem_blk; i++) { |
| 1591 |
iproc_offset += globals.Proc_Num_Elem_In_Blk[iproc][i]; |
| 1592 |
iproc_start += |
| 1593 |
globals.Proc_Num_Elem_In_Blk[iproc][i] * globals.Proc_Nodes_Per_Elem[iproc][i]; |
| 1594 |
} |
| 1595 |
|
| 1596 |
INT iglobal_begin = iglobal_offset + istart_elem; |
| 1597 |
INT iglobal_end = iglobal_begin + num_elem; |
| 1598 |
|
| 1599 |
size_t ibegin = iproc_offset; |
| 1600 |
size_t iend = iproc_offset + globals.Proc_Num_Elem_In_Blk[iproc][ielem_blk]; |
| 1601 |
|
| 1602 |
/* OPTIMIZE: Check that max globals.GElems is >min < max...*/ |
| 1603 |
/* Note that globals.GElems is sorted, so we can check that the |
| 1604 |
* ranges overlap and skip the loop if they don't */ |
| 1605 |
if (globals.GElems[iproc][ibegin] < iglobal_end && |
| 1606 |
globals.GElems[iproc][iend - 1] >= iglobal_begin) { |
| 1607 |
|
| 1608 |
for (size_t i = ibegin; i < iend; i++) { |
| 1609 |
INT iglobal_elem = globals.GElems[iproc][i]; |
| 1610 |
|
| 1611 |
/* globals.GElems is sorted, so if we are outside the iglobal range, |
| 1612 |
* break out of this loop... |
| 1613 |
*/ |
| 1614 |
if (iglobal_elem > iglobal_end) { |
| 1615 |
break; |
| 1616 |
} |
| 1617 |
|
| 1618 |
if (iglobal_begin <= iglobal_elem && iglobal_elem < iglobal_end) { |
| 1619 |
count_hits++; |
| 1620 |
|
| 1621 |
size_t ipos = |
| 1622 |
iproc_start + (i - iproc_offset) * globals.Proc_Nodes_Per_Elem[iproc][ielem_blk]; |
| 1623 |
|
| 1624 |
size_t iglobal_pos = |
| 1625 |
(iglobal_elem - iglobal_begin) * globals.Proc_Nodes_Per_Elem[iproc][ielem_blk]; |
| 1626 |
|
| 1627 |
assert(iglobal_pos < num_elem * globals.Proc_Nodes_Per_Elem[iproc][ielem_blk]); |
| 1628 |
/* |
| 1629 |
* Store the connectivity information for the current element into |
| 1630 |
* globals.Proc_Elem_Connect []. At the same time, Decrement by one the |
| 1631 |
* value of all nodes in the connectivity matrix. |
| 1632 |
*/ |
| 1633 |
for (int j = 0; j < globals.Proc_Nodes_Per_Elem[iproc][ielem_blk]; j++) { |
| 1634 |
globals.Proc_Elem_Connect[iproc][ipos++] = elem_blk[iglobal_pos++] - 1; |
| 1635 |
} |
| 1636 |
} |
| 1637 |
} |
| 1638 |
} |
| 1639 |
} /* END if (globals.Proc_Elem_Blk_Ids[ielem_blk]== ... */ |
| 1640 |
} |
| 1641 |
|
| 1642 |
if (!found) { |
| 1643 |
*local_ielem_blk = -1; |
| 1644 |
} |
| 1645 |
|
| 1646 |
return count_hits; |
| 1647 |
|
| 1648 |
} /* extract_elem_connect */ |
| 1649 |
|
| 1650 |
/*****************************************************************************/ |
| 1651 |
/*****************************************************************************/ |
| 1652 |
/*****************************************************************************/ |
| 1653 |
|
| 1654 |
template <typename T, typename INT> |
| 1655 |
void NemSpread<T, INT>::extract_elem_attr(T *elem_attr, int icurrent_elem_blk, size_t istart_elem, |
| 1656 |
size_t iend_elem, int natt_p_elem, int iproc) |
| 1657 |
{ |
| 1658 |
/* |
| 1659 |
* Function which extracts the element connectivity list from the current |
| 1660 |
* element connectivity message. The results are put into the global |
| 1661 |
* vector, globals.Proc_Elem_Connect. This function is run on all processors, |
| 1662 |
* include Proc = 0 |
| 1663 |
* |
| 1664 |
* Author: Scott Hutchinson (1421) |
| 1665 |
* |
| 1666 |
* ------------------------------------------------------------------ |
| 1667 |
* |
| 1668 |
* input |
| 1669 |
* ----------- |
| 1670 |
* elem_attr = Vector containing the element attributes for this message. |
| 1671 |
* This message consists of the either the whole or |
| 1672 |
* part of the element attribute array for a single |
| 1673 |
* element block. |
| 1674 |
* |
| 1675 |
* istart_elem = This is the starting value of the element number for |
| 1676 |
* the element attribute message. i.e., the first |
| 1677 |
* message in an element block will have |
| 1678 |
* istart_elem = 0. The second message for that |
| 1679 |
* element block will have the value: |
| 1680 |
* istart_elem = num_elem_per_message. |
| 1681 |
* iend_elem = This is the value + 1 of the last element number for |
| 1682 |
* the element attribute message. i.e., the first |
| 1683 |
* message in an element block will have |
| 1684 |
* iend_elem = num_elem_per_message |
| 1685 |
*/ |
| 1686 |
|
| 1687 |
/* Match the Element Block Id of the global block with the local block number */ |
| 1688 |
for (int ielem_blk = 0; ielem_blk < globals.Proc_Num_Elem_Blk[iproc]; ielem_blk++) { |
| 1689 |
if (globals.Proc_Elem_Blk_Ids[iproc][ielem_blk] == Elem_Blk_Ids[icurrent_elem_blk]) { |
| 1690 |
|
| 1691 |
/* Some elements in the current element block passed in from |
| 1692 |
read_elem_blk may be on this processor */ |
| 1693 |
|
| 1694 |
/* Calculate the number of elements in the current message */ |
| 1695 |
size_t num_elem = iend_elem - istart_elem; |
| 1696 |
|
| 1697 |
/* Calculate iglobal_offset - The sum of the elements in all the global |
| 1698 |
* element blocks preceding this one |
| 1699 |
*/ |
| 1700 |
size_t iglobal_offset = 0; |
| 1701 |
for (int i = 0; i < globals.GElem_Blks[iproc][ielem_blk]; i++) { |
| 1702 |
iglobal_offset += Num_Elem_In_Blk[i]; |
| 1703 |
} |
| 1704 |
|
| 1705 |
/* Calculate iproc_offset - The sum of the elements in all the |
| 1706 |
* processor's |
| 1707 |
* element blocks preceding this one. |
| 1708 |
* Calculate iproc_start - The starting position in the connectivity |
| 1709 |
* vector for this element block, |
| 1710 |
* globals.Proc_Elem_Connect[] |
| 1711 |
*/ |
| 1712 |
size_t iproc_offset = 0; |
| 1713 |
size_t iproc_start = 0; |
| 1714 |
for (int i = 0; i < ielem_blk; i++) { |
| 1715 |
iproc_offset += globals.Proc_Num_Elem_In_Blk[iproc][i]; |
| 1716 |
iproc_start += globals.Proc_Num_Elem_In_Blk[iproc][i] * globals.Proc_Num_Attr[iproc][i]; |
| 1717 |
} |
| 1718 |
|
| 1719 |
INT iglobal_begin = iglobal_offset + istart_elem; |
| 1720 |
INT iglobal_end = iglobal_begin + num_elem; |
| 1721 |
|
| 1722 |
size_t ibegin = iproc_offset; |
| 1723 |
size_t iend = iproc_offset + globals.Proc_Num_Elem_In_Blk[iproc][ielem_blk]; |
| 1724 |
|
| 1725 |
/* OPTIMIZE: Check that max globals.GElems is >min < max...*/ |
| 1726 |
/* Note that globals.GElems is sorted, so we can check that the |
| 1727 |
* ranges overlap and skip the loop if they don't */ |
| 1728 |
if (globals.GElems[iproc][ibegin] < iglobal_end && |
| 1729 |
globals.GElems[iproc][iend - 1] >= iglobal_begin) { |
| 1730 |
|
| 1731 |
for (size_t i = ibegin; i < iend; i++) { |
| 1732 |
INT iglobal_elem = globals.GElems[iproc][i]; |
| 1733 |
|
| 1734 |
/* globals.GElems is sorted, so if we are outside the iglobal range, |
| 1735 |
* break out of this loop... |
| 1736 |
*/ |
| 1737 |
if (iglobal_elem > iglobal_end) { |
| 1738 |
break; |
| 1739 |
} |
| 1740 |
|
| 1741 |
if (iglobal_begin <= iglobal_elem && iglobal_elem < iglobal_end) { |
| 1742 |
size_t ipos = iproc_start + (i - iproc_offset) * natt_p_elem; |
| 1743 |
size_t iglobal_pos = (iglobal_elem - iglobal_begin) * natt_p_elem; |
| 1744 |
|
| 1745 |
assert(iglobal_pos < num_elem * natt_p_elem); |
| 1746 |
/* |
| 1747 |
* Store the connectivity information for the current element into |
| 1748 |
* globals.Proc_Elem_Connect []. At the same time, Decrement by one the |
| 1749 |
* value of all nodes in the connectivity matrix. |
| 1750 |
*/ |
| 1751 |
for (int j = 0; j < natt_p_elem; j++) { |
| 1752 |
globals.Proc_Elem_Attr[iproc][ipos++] = elem_attr[iglobal_pos++]; |
| 1753 |
} |
| 1754 |
} |
| 1755 |
} |
| 1756 |
} |
| 1757 |
} /* END if (globals.Proc_Elem_Blk_Ids[ielem_blk]== ... */ |
| 1758 |
} |
| 1759 |
} /* extract_elem_connect */ |
| 1760 |
|
| 1761 |
/*****************************************************************************/ |
| 1762 |
/*****************************************************************************/ |
| 1763 |
/*****************************************************************************/ |
| 1764 |
|
| 1765 |
template <typename T, typename INT> |
| 1766 |
void NemSpread<T, INT>::find_elem_block(std::vector<INT> &proc_elem_blk, int iproc, |
| 1767 |
int /*proc_for*/) |
| 1768 |
|
| 1769 |
{ |
| 1770 |
/* Function which finds the element block which owns each element on the |
| 1771 |
* current processor. In addition, a map from the local element block number |
| 1772 |
* to the global element block number, *globals.GElem_Blks, is created. |
| 1773 |
* globals.Num_Elem_Blk, the number of element blocks on the current processor, |
| 1774 |
* is calculated This function is called by every processor. |
| 1775 |
* |
| 1776 |
* Author(s): Scott Hutchinson (9221) |
| 1777 |
* Gary Hennigan (9221) |
| 1778 |
* |
| 1779 |
* Output: |
| 1780 |
* |
| 1781 |
* globals.Num_Elem_Blk = Number of element blocks on the processor |
| 1782 |
* (Global int variable) |
| 1783 |
* |
| 1784 |
* *proc_elem_blk = Vector of element block ids for the local |
| 1785 |
* elements defined on the processor. |
| 1786 |
* (Local int vector of length |
| 1787 |
* globals.Num_Internal_Elems + globals.Num_Border_Elems) |
| 1788 |
* |
| 1789 |
* *globals.GElem_Blks = Map from the local element block number to |
| 1790 |
* the global element block number. |
| 1791 |
* (Global int vector of length globals.Num_Elem_Blk) |
| 1792 |
*/ |
| 1793 |
if (globals.Num_Elem_Blk == 0) { |
| 1794 |
return; |
| 1795 |
} |
| 1796 |
|
| 1797 |
/* Boolean vector of length globals.Num_Elem_Blk If the i'th |
| 1798 |
element block exists on the current processor, the i'th entry |
| 1799 |
is set to TRUE */ |
| 1800 |
std::vector<bool> elem_in_blk(globals.Num_Elem_Blk); |
| 1801 |
|
| 1802 |
/* Vector of integer offsets into the vector globals.GElems. |
| 1803 |
It has a length of globals.Num_Elem_Blk+1. The i'th entry |
| 1804 |
points to the beginning of of the element map information |
| 1805 |
for the first element in the i'th element block. The |
| 1806 |
(i+1)th entry points to the last element for the i'th block |
| 1807 |
in the vector globals.GElem. */ |
| 1808 |
std::vector<INT> elem_blk_point(globals.Num_Elem_Blk + 1); |
| 1809 |
|
| 1810 |
/* |
| 1811 |
* Construct a vector of pointers to the beginning of element blocks in an |
| 1812 |
* element connectivity list (vector) |
| 1813 |
*/ |
| 1814 |
elem_blk_point[0] = 0; |
| 1815 |
for (int i = 0; i < globals.Num_Elem_Blk; i++) { |
| 1816 |
elem_blk_point[i + 1] = elem_blk_point[i] + Num_Elem_In_Blk[i]; |
| 1817 |
} |
| 1818 |
|
| 1819 |
/* |
| 1820 |
* Find out which elements belonging to the current processor are in which |
| 1821 |
* local element block. Store this in *proc_elem_blk. |
| 1822 |
*/ |
| 1823 |
|
| 1824 |
/* Internal Elements */ |
| 1825 |
if (check_monot(&globals.GElems[iproc][0], globals.Num_Internal_Elems[iproc])) { |
| 1826 |
size_t tmp_cnt = globals.Num_Internal_Elems[iproc]; |
| 1827 |
int j = 0; |
| 1828 |
size_t i = 0; |
| 1829 |
while (i < tmp_cnt && j < globals.Num_Elem_Blk) { |
| 1830 |
while (i < tmp_cnt && globals.GElems[iproc][i] < elem_blk_point[j + 1]) { |
| 1831 |
assert(globals.GElems[iproc][i] >= elem_blk_point[j]); |
| 1832 |
proc_elem_blk[i++] = j; |
| 1833 |
elem_in_blk[j] = true; |
| 1834 |
} |
| 1835 |
j++; |
| 1836 |
} |
| 1837 |
} |
| 1838 |
else { |
| 1839 |
for (INT i = 0; i < globals.Num_Internal_Elems[iproc]; i++) { |
| 1840 |
bool found = false; |
| 1841 |
for (int j = 0; j < globals.Num_Elem_Blk && !found; j++) { |
| 1842 |
if (globals.GElems[iproc][i] < elem_blk_point[j + 1] && |
| 1843 |
globals.GElems[iproc][i] >= elem_blk_point[j]) { |
| 1844 |
proc_elem_blk[i] = j; |
| 1845 |
elem_in_blk[j] = true; |
| 1846 |
found = true; |
| 1847 |
} |
| 1848 |
} |
| 1849 |
if (!found) { |
| 1850 |
fmt::print(stderr, "find_elem_block: Error!:\n"); |
| 1851 |
fmt::print(stderr, |
| 1852 |
"\tElement {} not found in any element " |
| 1853 |
"block.\n", |
| 1854 |
i); |
| 1855 |
exit(1); |
| 1856 |
} |
| 1857 |
} |
| 1858 |
} |
| 1859 |
|
| 1860 |
/* Border Elements */ |
| 1861 |
if (check_monot(&globals.GElems[iproc][globals.Num_Internal_Elems[iproc]], |
| 1862 |
globals.Num_Border_Elems[iproc])) { |
| 1863 |
INT tmp_cnt = globals.Num_Internal_Elems[iproc] + globals.Num_Border_Elems[iproc]; |
| 1864 |
int j = 0; |
| 1865 |
INT i = globals.Num_Internal_Elems[iproc]; |
| 1866 |
while (i < tmp_cnt && j < globals.Num_Elem_Blk) { |
| 1867 |
while (i < tmp_cnt && globals.GElems[iproc][i] < elem_blk_point[j + 1]) { |
| 1868 |
assert(globals.GElems[iproc][i] >= elem_blk_point[j]); |
| 1869 |
proc_elem_blk[i++] = j; |
| 1870 |
elem_in_blk[j] = true; |
| 1871 |
} |
| 1872 |
j++; |
| 1873 |
} |
| 1874 |
} |
| 1875 |
else { |
| 1876 |
INT tmp_cnt = globals.Num_Internal_Elems[iproc] + globals.Num_Border_Elems[iproc]; |
| 1877 |
for (INT i = globals.Num_Internal_Elems[iproc]; i < tmp_cnt; i++) { |
| 1878 |
bool found = false; |
| 1879 |
for (int j = 0; j < globals.Num_Elem_Blk && !found; j++) { |
| 1880 |
if (globals.GElems[iproc][i] < elem_blk_point[j + 1] && |
| 1881 |
globals.GElems[iproc][i] >= elem_blk_point[j]) { |
| 1882 |
proc_elem_blk[i] = j; |
| 1883 |
elem_in_blk[j] = true; |
| 1884 |
found = true; |
| 1885 |
} |
| 1886 |
} |
| 1887 |
if (!found) { |
| 1888 |
fmt::print(stderr, "find_elem_block: Error!:\n"); |
| 1889 |
fmt::print(stderr, |
| 1890 |
"\tElement {} not found in any element " |
| 1891 |
"block.\n", |
| 1892 |
i); |
| 1893 |
exit(1); |
| 1894 |
} |
| 1895 |
} |
| 1896 |
} |
| 1897 |
|
| 1898 |
/* |
| 1899 |
* Reorder globals.GElems based on the element block information. This is necessary |
| 1900 |
* to insure that internal and border elements that are in the same element |
| 1901 |
* block are sequentially contained in globals.GElems, which is assumed in some |
| 1902 |
* later operations. |
| 1903 |
*/ |
| 1904 |
|
| 1905 |
my_sort(globals.Num_Internal_Elems[iproc] + globals.Num_Border_Elems[iproc], Data(proc_elem_blk), |
| 1906 |
Data(globals.GElems[iproc])); |
| 1907 |
|
| 1908 |
/* Now change proc_elem_blk to be a list of global element block IDs */ |
| 1909 |
for (INT i = 0; i < globals.Num_Internal_Elems[iproc] + globals.Num_Border_Elems[iproc]; i++) { |
| 1910 |
size_t j = proc_elem_blk[i]; |
| 1911 |
proc_elem_blk[i] = Elem_Blk_Ids[j]; |
| 1912 |
} |
| 1913 |
|
| 1914 |
/* Count the number of element blocks defined on this processor */ |
| 1915 |
globals.Proc_Num_Elem_Blk[iproc] = 0; |
| 1916 |
for (int i = 0; i < globals.Num_Elem_Blk; i++) { |
| 1917 |
if (elem_in_blk[i]) { |
| 1918 |
globals.Proc_Num_Elem_Blk[iproc]++; |
| 1919 |
} |
| 1920 |
} |
| 1921 |
|
| 1922 |
/* |
| 1923 |
* Create a map from the current processor's element block number to the |
| 1924 |
* 'global' element block number, globals.GElem_Blks. This is a permanent Global |
| 1925 |
* Map. |
| 1926 |
*/ |
| 1927 |
globals.GElem_Blks[iproc] = |
| 1928 |
(INT *)array_alloc(__FILE__, __LINE__, 1, globals.Num_Elem_Blk, sizeof(INT)); |
| 1929 |
|
| 1930 |
for (int i = 0, icount = 0; i < globals.Num_Elem_Blk; i++) { |
| 1931 |
if (elem_in_blk[i]) { |
| 1932 |
globals.GElem_Blks[iproc][icount++] = i; |
| 1933 |
} |
| 1934 |
} |
| 1935 |
} /* END of routine find_elem_block() ****************************************/ |
| 1936 |
/*****************************************************************************/ |
| 1937 |
/*****************************************************************************/ |
| 1938 |
/*****************************************************************************/ |
| 1939 |
|
| 1940 |
template <typename T, typename INT> |
| 1941 |
void NemSpread<T, INT>::read_node_sets(int exoid, std::vector<INT> &num_nodes_in_node_set, |
| 1942 |
std::vector<INT> &num_df_in_nsets) |
| 1943 |
|
| 1944 |
/* Function which reads the node sets information from an EXODUS database. |
| 1945 |
* It read information in chunks. Then, it broadcasts the chunk to all |
| 1946 |
* processors. Each processor, then searches for the chunk for what it is |
| 1947 |
* responsible for. This function is called by all processors. |
| 1948 |
* |
| 1949 |
* Global Variables which are set by this routine |
| 1950 |
* ----------------------------------------------- |
| 1951 |
* |
| 1952 |
* globals.Proc_Num_Node_Sets = Number of node sets on the current processor |
| 1953 |
* |
| 1954 |
* globals.GNode_Sets [globals.Proc_Num_Node_Sets] |
| 1955 |
* = Mapping between the local node set number and the |
| 1956 |
* global node set number. |
| 1957 |
* |
| 1958 |
* globals.S_Ids [globals.Proc_Num_Node_Sets] |
| 1959 |
* = Set IDs for the node sets defined on the current |
| 1960 |
* processor |
| 1961 |
* |
| 1962 |
* globals.S_Count [globals.Proc_Num_Node_Sets] |
| 1963 |
* = Number of nodes in the node sets defined on |
| 1964 |
* the current processor |
| 1965 |
* |
| 1966 |
* globals.S_Pointers [globals.Proc_Num_Node_Sets] |
| 1967 |
* = Pointers into the node set list for the node sets |
| 1968 |
* defined on the current processor. |
| 1969 |
* |
| 1970 |
* globals.NS_List_Length = Length of the local node set list. |
| 1971 |
* |
| 1972 |
* globals.NS_List [globals.NS_List_Length] |
| 1973 |
* = Concatenated node set list for the current |
| 1974 |
* processor. |
| 1975 |
*/ |
| 1976 |
|
| 1977 |
{ |
| 1978 |
size_t num_messages; |
| 1979 |
size_t num_left_over; |
| 1980 |
size_t num_node_per_message; |
| 1981 |
|
| 1982 |
/* Allocate arrays */ |
| 1983 |
std::vector<INT> list_length(Proc_Info[2]); |
| 1984 |
std::vector<INT> proc_num_ns(Proc_Info[2]); |
| 1985 |
std::vector<INT> ns_cntr(Proc_Info[2]); |
| 1986 |
std::vector<bool> ns_on_proc(Proc_Info[2]); |
| 1987 |
|
| 1988 |
/* pointers into the concatenated node set list which locate the start of the node sets */ |
| 1989 |
std::vector<std::vector<INT>> proc_list_pointer(Proc_Info[2]); |
| 1990 |
|
| 1991 |
/* mapping from the processor node set numbers to the global node set numbers */ |
| 1992 |
std::vector<std::vector<INT>> ns_proc2glob(Proc_Info[2]); |
| 1993 |
|
| 1994 |
/* the number of nodes in each processor's node set */ |
| 1995 |
std::vector<std::vector<INT>> proc_list_count(Proc_Info[2]); |
| 1996 |
|
| 1997 |
/* the number of df's in each processor's node set (usually either zero or proc_list_count)*/ |
| 1998 |
std::vector<std::vector<INT>> proc_list_df_count(Proc_Info[2]); |
| 1999 |
|
| 2000 |
/* the node number lists for the current node set for the current processor */ |
| 2001 |
std::vector<std::vector<std::vector<INT>>> proc_list(Proc_Info[2]); |
| 2002 |
|
| 2003 |
/* Vector of pointers to nodeset global nodeeset node map lists. |
| 2004 |
There is one pointer for each nodeset */ |
| 2005 |
std::vector<std::vector<std::vector<INT>>> proc_gmap_list(Proc_Info[2]); |
| 2006 |
|
| 2007 |
/* the dist. factors for the current processor */ |
| 2008 |
std::vector<std::vector<std::vector<T>>> proc_list_df(Proc_Info[2]); |
| 2009 |
|
| 2010 |
/* Initialize */ |
| 2011 |
for (int iproc = 0; iproc < Proc_Info[2]; iproc++) { |
| 2012 |
ns_proc2glob[iproc].resize(globals.Num_Node_Set); |
| 2013 |
proc_list_pointer[iproc].resize(globals.Num_Node_Set); |
| 2014 |
proc_list_count[iproc].resize(globals.Num_Node_Set); |
| 2015 |
proc_list_df_count[iproc].resize(globals.Num_Node_Set); |
| 2016 |
|
| 2017 |
proc_list[iproc].resize(globals.Num_Node_Set); |
| 2018 |
proc_gmap_list[iproc].resize(globals.Num_Node_Set); |
| 2019 |
proc_list_df[iproc].resize(globals.Num_Node_Set); |
| 2020 |
} |
| 2021 |
|
| 2022 |
std::vector<INT> node_proc_index(globals.Num_Node + 1); |
| 2023 |
for (int iproc = 0; iproc < Proc_Info[2]; iproc++) { |
| 2024 |
size_t size = globals.Num_Internal_Nodes[iproc] + globals.Num_Border_Nodes[iproc]; |
| 2025 |
for (size_t i = 0; i < size; i++) { |
| 2026 |
size_t ind = globals.GNodes[iproc][i]; |
| 2027 |
node_proc_index[ind]++; |
| 2028 |
} |
| 2029 |
} |
| 2030 |
|
| 2031 |
/* Now convert the node_proc_index counts to an index... */ |
| 2032 |
size_t index = 0; |
| 2033 |
for (size_t i = 0; i < globals.Num_Node; i++) { |
| 2034 |
size_t count = node_proc_index[i]; |
| 2035 |
node_proc_index[i] = index; |
| 2036 |
index += count; |
| 2037 |
} |
| 2038 |
node_proc_index[globals.Num_Node] = index; |
| 2039 |
|
| 2040 |
/* Create a map from 'global node id' to processor */ |
| 2041 |
std::vector<INT> gmap(index, -1); |
| 2042 |
|
| 2043 |
/* Fill it in.... |
| 2044 |
* Note that since nodes can be on multiple processors, we can't |
| 2045 |
* normally do a simple one-to-one map of node to its processor. |
| 2046 |
* |
| 2047 |
* The values in gmap[] from node_proc_index[node] to node_proc_index[node+1] |
| 2048 |
* contains the listing of the processors that node 'node' are on. |
| 2049 |
*/ |
| 2050 |
for (int iproc = 0; iproc < Proc_Info[2]; iproc++) { |
| 2051 |
size_t size = globals.Num_Internal_Nodes[iproc] + globals.Num_Border_Nodes[iproc]; |
| 2052 |
for (size_t i = 0; i < size; i++) { |
| 2053 |
size_t node = globals.GNodes[iproc][i]; |
| 2054 |
size_t beg = node_proc_index[node]; |
| 2055 |
size_t end = node_proc_index[node + 1]; |
| 2056 |
for (size_t j = beg; j < end; j++) { |
| 2057 |
if (gmap[j] == -1) { |
| 2058 |
gmap[j] = iproc; |
| 2059 |
break; |
| 2060 |
} |
| 2061 |
} |
| 2062 |
} |
| 2063 |
} |
| 2064 |
|
| 2065 |
/*-------------------------------------------------------------------------*/ |
| 2066 |
/* LOOP OVER THE NODE SETS */ |
| 2067 |
/*-------------------------------------------------------------------------*/ |
| 2068 |
|
| 2069 |
for (int i = 0; i < globals.Num_Node_Set; i++) { |
| 2070 |
if (num_nodes_in_node_set[i] > 0) { |
| 2071 |
|
| 2072 |
for (int iproc = 0; iproc < Proc_Info[2]; iproc++) { |
| 2073 |
proc_num_ns[iproc] = 0; |
| 2074 |
ns_cntr[iproc] = 0; |
| 2075 |
ns_on_proc[iproc] = false; |
| 2076 |
} |
| 2077 |
|
| 2078 |
size_t iss_size = sizeof(INT); |
| 2079 |
if (num_df_in_nsets[i] > 0) { |
| 2080 |
iss_size += sizeof(T); |
| 2081 |
} |
| 2082 |
|
| 2083 |
find_message_info(iss_size, num_nodes_in_node_set[i], &num_node_per_message, &num_messages, |
| 2084 |
&num_left_over); |
| 2085 |
|
| 2086 |
if (Debug_Flag > 1) { |
| 2087 |
fmt::print("\nMessage summary for Node Set number {}, with an ID of {}:\n", i, |
| 2088 |
Node_Set_Ids[i]); |
| 2089 |
fmt::print("\tNumber of messages need for node set = {}\n", num_messages); |
| 2090 |
fmt::print("\tNumber of node IDs and dist. factors per message = {}\n", |
| 2091 |
num_node_per_message); |
| 2092 |
fmt::print("\tLength of each message = {}\n", num_node_per_message * iss_size); |
| 2093 |
} |
| 2094 |
|
| 2095 |
/* pointers into 'node_set' for the nodes that are common to |
| 2096 |
both 'node_set' and the internal and border nodes for the |
| 2097 |
current processor */ |
| 2098 |
|
| 2099 |
std::vector<INT> node_set(num_node_per_message); |
| 2100 |
std::vector<T> node_set_df; |
| 2101 |
if (num_df_in_nsets[i] > 0) { |
| 2102 |
node_set_df.resize(num_node_per_message); |
| 2103 |
} |
| 2104 |
|
| 2105 |
/*---------------------------------------------------------------------*/ |
| 2106 |
/* LOOP OVER THE MESSAGES */ |
| 2107 |
/*---------------------------------------------------------------------*/ |
| 2108 |
|
| 2109 |
for (size_t imess = 0; imess < num_messages; imess++) { |
| 2110 |
|
| 2111 |
std::vector<INT> proc_ns_node_count(Proc_Info[2]); // Count nodes on this processor. |
| 2112 |
|
| 2113 |
size_t istart_ns = imess * num_node_per_message; |
| 2114 |
if (num_left_over != 0 && imess == num_messages - 1) { |
| 2115 |
num_node_per_message = num_left_over; |
| 2116 |
} |
| 2117 |
|
| 2118 |
/* Read in the part of the node set that will fit in the message */ |
| 2119 |
check_exodus_error(ex_get_partial_set(exoid, EX_NODE_SET, Node_Set_Ids[i], (istart_ns + 1), |
| 2120 |
num_node_per_message, Data(node_set), nullptr), |
| 2121 |
"ex_get_partial_set"); |
| 2122 |
|
| 2123 |
if (num_df_in_nsets[i] > 0) { |
| 2124 |
check_exodus_error(ex_get_partial_set_dist_fact(exoid, EX_NODE_SET, Node_Set_Ids[i], |
| 2125 |
(istart_ns + 1), num_node_per_message, |
| 2126 |
Data(node_set_df)), |
| 2127 |
"ex_get_partial_node_set_df"); |
| 2128 |
} |
| 2129 |
|
| 2130 |
/* Renumber nodes to start at node '0' instead of node '1' */ |
| 2131 |
for (size_t j = 0; j < num_node_per_message; j++) { |
| 2132 |
node_set[j]--; |
| 2133 |
} |
| 2134 |
|
| 2135 |
for (size_t j = 0; j < num_node_per_message; j++) { |
| 2136 |
size_t node = node_set[j]; |
| 2137 |
size_t beg = node_proc_index[node]; |
| 2138 |
size_t end = node_proc_index[node + 1]; |
| 2139 |
for (size_t k = beg; k < end; k++) { |
| 2140 |
proc_ns_node_count[gmap[k]]++; |
| 2141 |
} |
| 2142 |
} |
| 2143 |
|
| 2144 |
std::vector<std::vector<INT>> proc_ns_pointer(Proc_Info[2]); |
| 2145 |
for (int iproc = 0; iproc < Proc_Info[2]; iproc++) { |
| 2146 |
proc_ns_pointer[iproc].reserve(proc_ns_node_count[iproc]); |
| 2147 |
} |
| 2148 |
|
| 2149 |
for (size_t j = 0; j < num_node_per_message; j++) { |
| 2150 |
size_t node = node_set[j]; |
| 2151 |
size_t beg = node_proc_index[node]; |
| 2152 |
size_t end = node_proc_index[node + 1]; |
| 2153 |
for (size_t k = beg; k < end; k++) { |
| 2154 |
proc_ns_pointer[gmap[k]].push_back(j); |
| 2155 |
} |
| 2156 |
} |
| 2157 |
|
| 2158 |
/* Loop over the number of processors being handled */ |
| 2159 |
for (int iproc = 0; iproc < Proc_Info[2]; iproc++) { |
| 2160 |
/* |
| 2161 |
* If the message node set and either the Internal node or the Border |
| 2162 |
* node sets intersect, then add this intersection to the list |
| 2163 |
*/ |
| 2164 |
if (proc_ns_node_count[iproc] > 0) { |
| 2165 |
ns_on_proc[iproc] = true; |
| 2166 |
proc_num_ns[iproc] += proc_ns_node_count[iproc]; |
| 2167 |
|
| 2168 |
/* Allocate and store node information in a temporary vector */ |
| 2169 |
int nset = globals.Proc_Num_Node_Sets[iproc]; |
| 2170 |
proc_list[iproc][nset].resize(proc_num_ns[iproc]); |
| 2171 |
proc_gmap_list[iproc][nset].resize(proc_num_ns[iproc]); |
| 2172 |
if (num_df_in_nsets[i] > 0) { |
| 2173 |
proc_list_df[iproc][nset].resize(proc_num_ns[iproc]); |
| 2174 |
} |
| 2175 |
|
| 2176 |
for (INT j = 0; j < proc_ns_node_count[iproc]; j++) { |
| 2177 |
|
| 2178 |
proc_list[iproc][nset][ns_cntr[iproc]] = node_set[proc_ns_pointer[iproc][j]]; |
| 2179 |
proc_gmap_list[iproc][nset][ns_cntr[iproc]] = proc_ns_pointer[iproc][j] + istart_ns; |
| 2180 |
|
| 2181 |
if (num_df_in_nsets[i] > 0) { |
| 2182 |
proc_list_df[iproc][nset][ns_cntr[iproc]] = node_set_df[proc_ns_pointer[iproc][j]]; |
| 2183 |
} |
| 2184 |
|
| 2185 |
ns_cntr[iproc]++; |
| 2186 |
} |
| 2187 |
} /* End "if(intersection)" */ |
| 2188 |
} /* End "for(iproc=0; iproc <Proc_Info[2]; iproc++)" */ |
| 2189 |
} /* End "for(imess=0; imess < num_messages; imess++)" */ |
| 2190 |
|
| 2191 |
/* |
| 2192 |
* If any part of this node-set is on the processor, update the various |
| 2193 |
* pointers, lengths, etc. |
| 2194 |
*/ |
| 2195 |
for (int iproc = 0; iproc < Proc_Info[2]; iproc++) { |
| 2196 |
if (ns_on_proc[iproc]) { |
| 2197 |
ns_proc2glob[iproc][globals.Proc_Num_Node_Sets[iproc]] = i; |
| 2198 |
|
| 2199 |
proc_list_pointer[iproc][globals.Proc_Num_Node_Sets[iproc]] = list_length[iproc]; |
| 2200 |
|
| 2201 |
proc_list_count[iproc][globals.Proc_Num_Node_Sets[iproc]] = proc_num_ns[iproc]; |
| 2202 |
|
| 2203 |
if (num_df_in_nsets[i] > 0) { |
| 2204 |
proc_list_df_count[iproc][globals.Proc_Num_Node_Sets[iproc]] = proc_num_ns[iproc]; |
| 2205 |
} |
| 2206 |
else { |
| 2207 |
proc_list_df_count[iproc][globals.Proc_Num_Node_Sets[iproc]] = 0; |
| 2208 |
} |
| 2209 |
|
| 2210 |
(globals.Proc_Num_Node_Sets[iproc])++; |
| 2211 |
|
| 2212 |
list_length[iproc] += proc_num_ns[iproc]; |
| 2213 |
} |
| 2214 |
} |
| 2215 |
} /* END "if (num_nodes_in_node_set[i] > 0)" */ |
| 2216 |
} /* END "for (i = 0; i < globals.Num_Node_Set; i++)" */ |
| 2217 |
|
| 2218 |
/*--------------------------------------------------------------------------*/ |
| 2219 |
/* WRITE PERMANENT ARRAYS FOR NODE SET INFO */ |
| 2220 |
/*--------------------------------------------------------------------------*/ |
| 2221 |
|
| 2222 |
for (int iproc = 0; iproc < Proc_Info[2]; iproc++) { |
| 2223 |
|
| 2224 |
/* Allocate Permanent Arrays for node sets in one long malloc */ |
| 2225 |
if (globals.Num_Node_Set > 0) { |
| 2226 |
|
| 2227 |
/* |
| 2228 |
* Note that the alloc is done based on globals.Num_Node_Set, rather than |
| 2229 |
* globals.Proc_Num_Node_Sets[] due to the fact that nullptr entities are |
| 2230 |
* stored on processors not having a particular node set. |
| 2231 |
*/ |
| 2232 |
globals.Proc_NS_Ids[iproc] = (INT *)array_alloc( |
| 2233 |
__FILE__, __LINE__, 1, |
| 2234 |
(3 * globals.Num_Node_Set + 2 * globals.Proc_Num_Node_Sets[iproc] + list_length[iproc]), |
| 2235 |
sizeof(INT)); |
| 2236 |
|
| 2237 |
globals.Proc_NS_Count[iproc] = globals.Proc_NS_Ids[iproc] + globals.Num_Node_Set; |
| 2238 |
globals.Proc_NS_DF_Count[iproc] = globals.Proc_NS_Count[iproc] + globals.Num_Node_Set; |
| 2239 |
globals.Proc_NS_Pointers[iproc] = globals.Proc_NS_DF_Count[iproc] + globals.Num_Node_Set; |
| 2240 |
globals.GNode_Sets[iproc] = |
| 2241 |
globals.Proc_NS_Pointers[iproc] + globals.Proc_Num_Node_Sets[iproc]; |
| 2242 |
globals.Proc_NS_List[iproc] = globals.GNode_Sets[iproc] + globals.Proc_Num_Node_Sets[iproc]; |
| 2243 |
|
| 2244 |
globals.Proc_NS_GNMap_List[iproc] = |
| 2245 |
(INT *)array_alloc(__FILE__, __LINE__, 1, list_length[iproc], sizeof(INT)); |
| 2246 |
|
| 2247 |
if (list_length[iproc] > 0) { |
| 2248 |
globals.Proc_NS_Dist_Fact[iproc].resize(list_length[iproc]); |
| 2249 |
} |
| 2250 |
} |
| 2251 |
else { |
| 2252 |
globals.Proc_NS_Ids[iproc] = nullptr; |
| 2253 |
globals.Proc_NS_Count[iproc] = nullptr; |
| 2254 |
globals.Proc_NS_Pointers[iproc] = nullptr; |
| 2255 |
globals.Proc_NS_List[iproc] = nullptr; |
| 2256 |
} |
| 2257 |
|
| 2258 |
/* |
| 2259 |
* Fill in the permanent node set arrays which have length, |
| 2260 |
* globals.Proc_Num_Node_Sets, the total number of node sets defined on the |
| 2261 |
* current processor. |
| 2262 |
*/ |
| 2263 |
globals.Proc_NS_List_Length[iproc] = 0; |
| 2264 |
for (int i = 0; i < globals.Proc_Num_Node_Sets[iproc]; i++) { |
| 2265 |
globals.GNode_Sets[iproc][i] = ns_proc2glob[iproc][i]; |
| 2266 |
globals.Proc_NS_Ids[iproc][i] = Node_Set_Ids[ns_proc2glob[iproc][i]]; |
| 2267 |
globals.Proc_NS_Count[iproc][i] = proc_list_count[iproc][i]; |
| 2268 |
globals.Proc_NS_Pointers[iproc][i] = proc_list_pointer[iproc][i]; |
| 2269 |
globals.Proc_NS_List_Length[iproc] += globals.Proc_NS_Count[iproc][i]; |
| 2270 |
globals.Proc_NS_DF_Count[iproc][i] = proc_list_df_count[iproc][i]; |
| 2271 |
} |
| 2272 |
|
| 2273 |
/* Construct the concatenated node list */ |
| 2274 |
for (int i = 0; i < globals.Proc_Num_Node_Sets[iproc]; i++) { |
| 2275 |
for (INT j = 0; j < globals.Proc_NS_Count[iproc][i]; j++) { |
| 2276 |
globals.Proc_NS_List[iproc][globals.Proc_NS_Pointers[iproc][i] + j] = |
| 2277 |
proc_list[iproc][i][j]; |
| 2278 |
globals.Proc_NS_GNMap_List[iproc][globals.Proc_NS_Pointers[iproc][i] + j] = |
| 2279 |
proc_gmap_list[iproc][i][j]; |
| 2280 |
|
| 2281 |
if (globals.Proc_NS_DF_Count[iproc][i] > 0) { |
| 2282 |
globals.Proc_NS_Dist_Fact[iproc][globals.Proc_NS_Pointers[iproc][i] + j] = |
| 2283 |
proc_list_df[iproc][i][j]; |
| 2284 |
} |
| 2285 |
} |
| 2286 |
} |
| 2287 |
} /* End "for(iproc=0; iproc <Proc_Info[2]; iproc++)" */ |
| 2288 |
|
| 2289 |
} /* END of routine read_node_sets () ****************************************/ |
| 2290 |
|
| 2291 |
/*****************************************************************************/ |
| 2292 |
/*****************************************************************************/ |
| 2293 |
/*****************************************************************************/ |
| 2294 |
|
| 2295 |
template <typename T, typename INT> |
| 2296 |
void NemSpread<T, INT>::read_side_sets(int exoid, std::vector<INT> &num_elem_in_ssets, |
| 2297 |
std::vector<INT> &num_df_in_ssets) |
| 2298 |
{ |
| 2299 |
/* |
| 2300 |
* Function which reads the side sets information from an EXODUS database |
| 2301 |
* for a given processor. It then broadcasts all information to every |
| 2302 |
* processor. Each processor extracts and keeps only the information that it |
| 2303 |
* needs to run its local problem. |
| 2304 |
* |
| 2305 |
* Global Variables which are set by this routine |
| 2306 |
* ----------------------------------------------- |
| 2307 |
* |
| 2308 |
* globals.Proc_Num_Side_Sets = Number of side sets on the current processor |
| 2309 |
* |
| 2310 |
* globals.GSide_Sets [globals.Proc_Num_Side_Sets] |
| 2311 |
* = Mapping between the local side set number and the |
| 2312 |
* global side set number. |
| 2313 |
* |
| 2314 |
* globals.S_Ids [globals.Proc_Num_Side_Sets] |
| 2315 |
* = Set IDs for the side sets defined on the current |
| 2316 |
* processor |
| 2317 |
* |
| 2318 |
* globals.Proc_SS_Elem_Count [globals.Proc_Num_Side_Sets] |
| 2319 |
* = Number of elements in the side set for the |
| 2320 |
* current processor |
| 2321 |
* |
| 2322 |
* globals.S_Side_Count [globals.Proc_Num_Side_Sets] |
| 2323 |
* = Number of nodes in the node set list for the side |
| 2324 |
* set for the current processor. |
| 2325 |
* |
| 2326 |
* globals.Proc_SS_Elem_Pointers [globals.Proc_Num_Side_Sets] |
| 2327 |
* = Side sets pointer record for elements for |
| 2328 |
* the side sets in a given processor |
| 2329 |
* |
| 2330 |
* globals.Proc_SS_Elem_List_Length |
| 2331 |
* = Length of the element list length for side sets |
| 2332 |
* on the current processor |
| 2333 |
* |
| 2334 |
* globals.Proc_SS_Elem_List [globals.Proc_SS_Elem_List_Length] |
| 2335 |
* = Concatenated vector of elements that comprise |
| 2336 |
* the side set definitions on the current |
| 2337 |
* processor. |
| 2338 |
* |
| 2339 |
* globals.S_Side_List [globals.Proc_SS_Elem_List_Length] |
| 2340 |
* = Concatenated vector of sides that comprise |
| 2341 |
* the side set definitions on the current |
| 2342 |
* processor. |
| 2343 |
*/ |
| 2344 |
|
| 2345 |
/* Vector of pointers to side-set element lists. There is one pointer for each side-set */ |
| 2346 |
std::vector<std::vector<std::vector<INT>>> proc_elem_list(Proc_Info[2]); |
| 2347 |
|
| 2348 |
/* Vector of pointers to side-set side lists. There is one pointer for each side-set */ |
| 2349 |
std::vector<std::vector<std::vector<INT>>> proc_side_list(Proc_Info[2]); |
| 2350 |
|
| 2351 |
/* Vector of pointers to side-set global sideset element map lists. There is one pointer for each |
| 2352 |
* side-set */ |
| 2353 |
std::vector<std::vector<std::vector<INT>>> proc_gmap_list(Proc_Info[2]); |
| 2354 |
|
| 2355 |
/* Vector of pointers to side-set df pointers There is one pointer for each side-set */ |
| 2356 |
std::vector<std::vector<std::vector<INT>>> proc_df_ptr(Proc_Info[2]); |
| 2357 |
std::vector<std::vector<std::vector<INT>>> proc_df_indx(Proc_Info[2]); |
| 2358 |
std::vector<std::vector<std::vector<INT>>> proc_df_indx_cnt(Proc_Info[2]); |
| 2359 |
|
| 2360 |
std::vector<std::vector<std::vector<T>>> proc_ss_df(Proc_Info[2]); |
| 2361 |
|
| 2362 |
std::vector<INT> ss_elem_cntr(Proc_Info[2]); |
| 2363 |
std::vector<bool> ss_on_proc(Proc_Info[2]); /* Flag to indicate that the ss is on the proc */ |
| 2364 |
std::vector<INT> elem_list_length(Proc_Info[2]); /* length of the element side-set list for the |
| 2365 |
current processor */ |
| 2366 |
std::vector<INT> proc_num_sides_ss(Proc_Info[2]); /* Number of sides in the current side set that |
| 2367 |
exist for the current processor */ |
| 2368 |
std::vector<INT> ntotal(Proc_Info[2]); |
| 2369 |
std::vector<INT> ss_num(Proc_Info[2]); |
| 2370 |
|
| 2371 |
/* Allocate temporary arrays */ |
| 2372 |
/* Array of counts of the dist. factors. */ |
| 2373 |
std::vector<std::vector<INT>> proc_ss_df_cnt(Proc_Info[2]); |
| 2374 |
|
| 2375 |
/* mapping from the processor side-set numbers to the global side-set numbers */ |
| 2376 |
std::vector<std::vector<INT>> ss_proc2glob(Proc_Info[2]); |
| 2377 |
|
| 2378 |
/* pointers into the concatenated element list which locate the start of the element lists */ |
| 2379 |
std::vector<std::vector<INT>> proc_elem_list_ptr(Proc_Info[2]); |
| 2380 |
|
| 2381 |
/* the number of elements in each processor's element set */ |
| 2382 |
std::vector<std::vector<INT>> proc_elem_list_cnt(Proc_Info[2]); |
| 2383 |
|
| 2384 |
for (int iproc = 0; iproc < Proc_Info[2]; iproc++) { |
| 2385 |
proc_elem_list[iproc].resize(globals.Num_Side_Set); |
| 2386 |
proc_side_list[iproc].resize(globals.Num_Side_Set); |
| 2387 |
proc_df_ptr[iproc].resize(globals.Num_Side_Set); |
| 2388 |
proc_df_indx[iproc].resize(globals.Num_Side_Set); |
| 2389 |
proc_df_indx_cnt[iproc].resize(globals.Num_Side_Set); |
| 2390 |
proc_gmap_list[iproc].resize(globals.Num_Side_Set); |
| 2391 |
|
| 2392 |
proc_ss_df[iproc].resize(globals.Num_Side_Set); |
| 2393 |
|
| 2394 |
ss_proc2glob[iproc].resize(globals.Num_Side_Set); |
| 2395 |
proc_elem_list_ptr[iproc].resize(globals.Num_Side_Set); |
| 2396 |
proc_elem_list_cnt[iproc].resize(globals.Num_Side_Set); |
| 2397 |
proc_ss_df_cnt[iproc].resize(globals.Num_Side_Set); |
| 2398 |
} |
| 2399 |
|
| 2400 |
/* Create a map from 'global element id' to processor */ |
| 2401 |
std::vector<INT> gmap(globals.Num_Elem, -1); |
| 2402 |
|
| 2403 |
/* Fill it in.... */ |
| 2404 |
for (int iproc = 0; iproc < Proc_Info[2]; iproc++) { |
| 2405 |
size_t size = globals.Num_Internal_Elems[iproc] + globals.Num_Border_Elems[iproc]; |
| 2406 |
for (size_t i = 0; i < size; i++) { |
| 2407 |
size_t ind = globals.GElems[iproc][i]; |
| 2408 |
gmap[ind] = iproc; |
| 2409 |
} |
| 2410 |
} |
| 2411 |
|
| 2412 |
/*-------------------------------------------------------------------------*/ |
| 2413 |
/* LOOP OVER THE SIDE SETS */ |
| 2414 |
/*-------------------------------------------------------------------------*/ |
| 2415 |
for (int i = 0; i < globals.Num_Side_Set; i++) { |
| 2416 |
|
| 2417 |
if (num_elem_in_ssets[i] > 0) { |
| 2418 |
|
| 2419 |
size_t ilast = 0; |
| 2420 |
int ilast_side = 0; |
| 2421 |
|
| 2422 |
for (int iproc = 0; iproc < Proc_Info[2]; iproc++) { |
| 2423 |
ss_on_proc[iproc] = false; |
| 2424 |
proc_num_sides_ss[iproc] = 0; |
| 2425 |
ss_elem_cntr[iproc] = 0; |
| 2426 |
} |
| 2427 |
|
| 2428 |
/* One element ID + One side ID */ |
| 2429 |
size_t iss_size = 3 * sizeof(INT); |
| 2430 |
|
| 2431 |
size_t num_messages; |
| 2432 |
size_t num_left_over; |
| 2433 |
size_t num_elem_per_message; |
| 2434 |
find_message_info(iss_size, num_elem_in_ssets[i], &num_elem_per_message, &num_messages, |
| 2435 |
&num_left_over); |
| 2436 |
|
| 2437 |
if (Debug_Flag >= 2) { |
| 2438 |
fmt::print("Message summary for Side Set number {}, with an ID of {}:\n", i, |
| 2439 |
Side_Set_Ids[i]); |
| 2440 |
fmt::print("\tNumber of messages needed for element and side list = {}\n", num_messages); |
| 2441 |
fmt::print("\tNumber of element and side IDs per message = {}\n", num_elem_per_message); |
| 2442 |
fmt::print("\tLength of each message = {}\n", iss_size * num_elem_per_message); |
| 2443 |
} |
| 2444 |
|
| 2445 |
/* Allocate temporary storage for the current message for the |
| 2446 |
* current side set on each proc |
| 2447 |
*/ |
| 2448 |
std::vector<INT> ss_elem_list(num_elem_per_message); /* side-set element list */ |
| 2449 |
std::vector<INT> ss_side_list(num_elem_per_message); /* side-set node list */ |
| 2450 |
std::vector<INT> ss_df_ptr(num_elem_per_message); /* pointer into the dist. factor list */ |
| 2451 |
|
| 2452 |
/*---------------------------------------------------------------------*/ |
| 2453 |
/* LOOP OVER THE MESSAGES */ |
| 2454 |
/*---------------------------------------------------------------------*/ |
| 2455 |
for (size_t imess = 0; imess < num_messages; imess++) { |
| 2456 |
|
| 2457 |
if (Debug_Flag >= 2) { |
| 2458 |
fmt::print("\tside set message: {} of {}\n", imess + 1, num_messages); |
| 2459 |
} |
| 2460 |
|
| 2461 |
size_t istart_ss = imess * num_elem_per_message; |
| 2462 |
|
| 2463 |
if (num_left_over != 0 && imess == num_messages - 1) { |
| 2464 |
num_elem_per_message = num_left_over; |
| 2465 |
} |
| 2466 |
|
| 2467 |
/* Read in the part of the side set that will fit in the message. */ |
| 2468 |
|
| 2469 |
check_exodus_error(ex_get_partial_set(exoid, EX_SIDE_SET, Side_Set_Ids[i], (istart_ss + 1), |
| 2470 |
num_elem_per_message, Data(ss_elem_list), |
| 2471 |
Data(ss_side_list)), |
| 2472 |
"ex_get_partial_set"); |
| 2473 |
|
| 2474 |
/* Fill in the distribution factor pointer vector */ |
| 2475 |
if (imess == 0) { |
| 2476 |
ilast = 0; |
| 2477 |
ilast_side = 0; |
| 2478 |
} |
| 2479 |
|
| 2480 |
ss_df_ptr[0] = ilast + ilast_side; |
| 2481 |
|
| 2482 |
for (size_t j = 1; j < num_elem_per_message; j++) { |
| 2483 |
|
| 2484 |
/* |
| 2485 |
* Kluge for "special" sidesest which has a single distribution |
| 2486 |
* factor for each element instead of 1 per face node.. |
| 2487 |
*/ |
| 2488 |
if (num_elem_in_ssets[i] == num_df_in_ssets[i]) { |
| 2489 |
ilast_side = 1; |
| 2490 |
} |
| 2491 |
else { |
| 2492 |
|
| 2493 |
ilast_side = |
| 2494 |
elem_info(NN_SIDE, GM_Elem_Types[(ss_elem_list[j - 1]) - 1], ss_side_list[j - 1]); |
| 2495 |
|
| 2496 |
/* |
| 2497 |
* kludge for HEXSHELL's |
| 2498 |
* where distribution factors are concerned, only count |
| 2499 |
* 4 nodes on the 6 node faces (they are just like HEX's) |
| 2500 |
*/ |
| 2501 |
if (GM_Elem_Types[(ss_elem_list[j - 1]) - 1] == HEXSHELL) { |
| 2502 |
ilast_side = 4; |
| 2503 |
} |
| 2504 |
} |
| 2505 |
|
| 2506 |
ss_df_ptr[j] = ss_df_ptr[j - 1] + ilast_side; |
| 2507 |
ilast = ss_df_ptr[j]; |
| 2508 |
} |
| 2509 |
|
| 2510 |
/* Renumber elements to start at '0' instead of '1' */ |
| 2511 |
for (size_t j = 0; j < num_elem_per_message; ss_elem_list[j++]--) { |
| 2512 |
; |
| 2513 |
} |
| 2514 |
|
| 2515 |
#if 1 |
| 2516 |
/* This method may be slightly faster, but uses lots of push backs |
| 2517 |
that may be bad in the long wrong. Keep both sets of code in case |
| 2518 |
this turns out to be bad... The other branch is also faster than |
| 2519 |
the older code, so it isn't bad to turn it on instead of this... */ |
| 2520 |
|
| 2521 |
/* Map the ss_elem_list elements to the processor that they are on */ |
| 2522 |
/* Note that an element may be in ss_elem_list multiple times */ |
| 2523 |
for (size_t j = 0; j < num_elem_per_message; j++) { |
| 2524 |
int iproc = gmap[ss_elem_list[j]]; |
| 2525 |
int sset = globals.Proc_Num_Side_Sets[iproc]; |
| 2526 |
proc_elem_list[iproc][sset].push_back(ss_elem_list[j]); |
| 2527 |
proc_side_list[iproc][sset].push_back(ss_side_list[j]); |
| 2528 |
proc_gmap_list[iproc][sset].push_back(j + istart_ss); |
| 2529 |
proc_df_ptr[iproc][sset].push_back(ss_df_ptr[j]); |
| 2530 |
} |
| 2531 |
|
| 2532 |
for (int iproc = 0; iproc < Proc_Info[2]; iproc++) { |
| 2533 |
if (!proc_elem_list[iproc][globals.Proc_Num_Side_Sets[iproc]].empty()) { |
| 2534 |
ss_on_proc[iproc] = true; |
| 2535 |
proc_num_sides_ss[iproc] = |
| 2536 |
proc_elem_list[iproc][globals.Proc_Num_Side_Sets[iproc]].size(); |
| 2537 |
} |
| 2538 |
} /* End "for(iproc=0; iproc <Proc_Info[2]; iproc++)" */ |
| 2539 |
#else |
| 2540 |
/* Map the ss_elem_list elements to the processor that they are on */ |
| 2541 |
/* Note that an element may be in ss_elem_list multiple times */ |
| 2542 |
|
| 2543 |
/* pointers into the global element side-set list for elements |
| 2544 |
which are common to the processor and the side-set */ |
| 2545 |
std::vector<std::vector<INT>> proc_es_pointer(Proc_Info[2]); |
| 2546 |
|
| 2547 |
for (size_t j = 0; j < num_elem_per_message; j++) { |
| 2548 |
int iproc = gmap[ss_elem_list[j]]; |
| 2549 |
int sset = globals.Proc_Num_Side_Sets[iproc]; |
| 2550 |
proc_es_pointer[iproc].push_back(j); |
| 2551 |
} |
| 2552 |
|
| 2553 |
for (int iproc = 0; iproc < Proc_Info[2]; iproc++) { |
| 2554 |
|
| 2555 |
/* |
| 2556 |
* Find the intersection between the elements in the side set and the |
| 2557 |
* elements owned by the current processor. |
| 2558 |
*/ |
| 2559 |
|
| 2560 |
if (!proc_es_pointer[iproc].empty()) { |
| 2561 |
proc_num_sides_ss[iproc] += proc_es_pointer[iproc].size(); |
| 2562 |
ss_on_proc[iproc] = true; |
| 2563 |
|
| 2564 |
/* |
| 2565 |
* Store the information for the current side set in temporary arrays. |
| 2566 |
* |
| 2567 |
* This part of the routine only gets executed if the side set is |
| 2568 |
* defined on the current processor. |
| 2569 |
*/ |
| 2570 |
|
| 2571 |
/* Allocate and store element information in a temporary vector */ |
| 2572 |
int sset = globals.Proc_Num_Side_Sets[iproc]; |
| 2573 |
proc_elem_list[iproc][sset].resize(proc_num_sides_ss[iproc]); |
| 2574 |
proc_side_list[iproc][sset].resize(proc_num_sides_ss[iproc]); |
| 2575 |
proc_gmap_list[iproc][sset].resize(proc_num_sides_ss[iproc]); |
| 2576 |
proc_df_ptr[iproc][sset].resize(proc_num_sides_ss[iproc]); |
| 2577 |
|
| 2578 |
/* Transfer the element, side number, and df pointers into "permanent" storage... */ |
| 2579 |
for (size_t j = 0; j < proc_es_pointer[iproc].size(); j++) { |
| 2580 |
proc_elem_list[iproc][sset][ss_elem_cntr[iproc]] = |
| 2581 |
ss_elem_list[proc_es_pointer[iproc][j]]; |
| 2582 |
proc_side_list[iproc][sset][ss_elem_cntr[iproc]] = |
| 2583 |
ss_side_list[proc_es_pointer[iproc][j]]; |
| 2584 |
proc_gmap_list[iproc][sset][ss_elem_cntr[iproc]] = |
| 2585 |
proc_es_pointer[iproc][j] + istart_ss; |
| 2586 |
proc_df_ptr[iproc][sset][(ss_elem_cntr[iproc])++] = |
| 2587 |
ss_df_ptr[proc_es_pointer[iproc][j]]; |
| 2588 |
} |
| 2589 |
} /* End "if(ipos_elem > 0)" */ |
| 2590 |
} /* End "for(iproc=0; iproc <Proc_Info[2]; iproc++)" */ |
| 2591 |
#endif |
| 2592 |
} /* End "for(imess=0; imess < num_messages; imess++)" */ |
| 2593 |
|
| 2594 |
/* Entire sideset has been read at this point */ |
| 2595 |
/* |
| 2596 |
* If any part of this side-set is on the processor, update the various |
| 2597 |
* pointers, lengths, etc. |
| 2598 |
*/ |
| 2599 |
|
| 2600 |
for (int iproc = 0; iproc < Proc_Info[2]; iproc++) { |
| 2601 |
if (ss_on_proc[iproc]) { |
| 2602 |
ss_proc2glob[iproc][globals.Proc_Num_Side_Sets[iproc]] = i; |
| 2603 |
|
| 2604 |
proc_elem_list_ptr[iproc][globals.Proc_Num_Side_Sets[iproc]] = elem_list_length[iproc]; |
| 2605 |
|
| 2606 |
proc_elem_list_cnt[iproc][globals.Proc_Num_Side_Sets[iproc]] = proc_num_sides_ss[iproc]; |
| 2607 |
|
| 2608 |
(globals.Proc_Num_Side_Sets[iproc])++; |
| 2609 |
elem_list_length[iproc] += proc_num_sides_ss[iproc]; |
| 2610 |
} |
| 2611 |
} |
| 2612 |
|
| 2613 |
/* Process any distribution factors in the side set */ |
| 2614 |
if (num_df_in_ssets[i] > 0) { |
| 2615 |
|
| 2616 |
iss_size = sizeof(T); |
| 2617 |
find_message_info(iss_size, num_df_in_ssets[i], &num_elem_per_message, &num_messages, |
| 2618 |
&num_left_over); |
| 2619 |
|
| 2620 |
if (Debug_Flag >= 4) { |
| 2621 |
fmt::print("Message summary for Side Set number {}, with ID of {}:\n", i, |
| 2622 |
Side_Set_Ids[i]); |
| 2623 |
fmt::print("\tNumber of messages needed for distribution " |
| 2624 |
"factors = {}\n", |
| 2625 |
num_messages); |
| 2626 |
fmt::print("\tNumber of dist. factors in each message = {}\n", num_elem_per_message); |
| 2627 |
fmt::print("\tLength of each message = {}\n", (num_elem_per_message * sizeof(T))); |
| 2628 |
} |
| 2629 |
|
| 2630 |
std::vector<T> ss_dist_fact(num_elem_per_message); /* side-set distribution factors */ |
| 2631 |
|
| 2632 |
/* set up an array to help speed up the searches below */ |
| 2633 |
std::vector<std::vector<INT>> proc_df_map(Proc_Info[2]); |
| 2634 |
|
| 2635 |
/* Loop over the messages needed for the distribution factors */ |
| 2636 |
for (size_t imess = 0; imess < num_messages; imess++) { |
| 2637 |
|
| 2638 |
size_t istart_ss = imess * num_elem_per_message; |
| 2639 |
if (num_left_over != 0 && imess == (num_messages - 1)) { |
| 2640 |
num_elem_per_message = num_left_over; |
| 2641 |
} |
| 2642 |
|
| 2643 |
/* Read in the part of the side set df's that will fit in the msg. */ |
| 2644 |
check_exodus_error(ex_get_partial_set_dist_fact(exoid, EX_SIDE_SET, Side_Set_Ids[i], |
| 2645 |
(istart_ss + 1), num_elem_per_message, |
| 2646 |
Data(ss_dist_fact)), |
| 2647 |
"ex_get_partial_set_dist_fact"); |
| 2648 |
|
| 2649 |
/* |
| 2650 |
* At this point a processor has the list of global element IDs |
| 2651 |
* that it owns that are contained in the side set. It also has |
| 2652 |
* a vector of pointers into the distribution factor vector telling |
| 2653 |
* where the distribution factors for a particular element in |
| 2654 |
* the side set begin. |
| 2655 |
*/ |
| 2656 |
|
| 2657 |
/* |
| 2658 |
* First figure out how many df's there are for this side set |
| 2659 |
* on this processor. |
| 2660 |
*/ |
| 2661 |
for (int iproc = 0; iproc < Proc_Info[2]; iproc++) { |
| 2662 |
|
| 2663 |
if (imess == 0) { |
| 2664 |
ntotal[iproc] = 0; |
| 2665 |
} |
| 2666 |
|
| 2667 |
if (imess == 0 && ss_on_proc[iproc]) { |
| 2668 |
|
| 2669 |
ss_num[iproc] = globals.Proc_Num_Side_Sets[iproc]; |
| 2670 |
ntotal[iproc] = 0; |
| 2671 |
|
| 2672 |
size_t indx = 0; |
| 2673 |
int ntotal_s = 0; // Nodes per element side... |
| 2674 |
|
| 2675 |
size_t cnt = proc_elem_list_cnt[iproc][ss_num[iproc] - 1]; |
| 2676 |
proc_df_map[iproc].resize(cnt); |
| 2677 |
proc_df_indx[iproc][ss_num[iproc] - 1].resize(cnt); |
| 2678 |
proc_df_indx_cnt[iproc][ss_num[iproc] - 1].resize(cnt); |
| 2679 |
|
| 2680 |
for (INT j = 0; j < proc_elem_list_cnt[iproc][ss_num[iproc] - 1]; j++) { |
| 2681 |
|
| 2682 |
size_t loc_elem = proc_elem_list[iproc][ss_num[iproc] - 1][j]; |
| 2683 |
size_t loc_side = proc_side_list[iproc][ss_num[iproc] - 1][j]; |
| 2684 |
|
| 2685 |
/* |
| 2686 |
* Kluge for "special" sidesest which has a single distribution |
| 2687 |
* factor for each element instead of 1 per face node.. |
| 2688 |
*/ |
| 2689 |
if (num_elem_in_ssets[i] == num_df_in_ssets[i]) { |
| 2690 |
ntotal_s = 1; |
| 2691 |
} |
| 2692 |
else { |
| 2693 |
|
| 2694 |
ntotal_s = elem_info(NN_SIDE, GM_Elem_Types[loc_elem], loc_side); |
| 2695 |
|
| 2696 |
/* |
| 2697 |
* kludge for HEXSHELL's |
| 2698 |
* where distribution factors are concerned, only count |
| 2699 |
* 4 nodes on the 6 node faces (they are just like HEX's) |
| 2700 |
*/ |
| 2701 |
if (GM_Elem_Types[loc_elem] == HEXSHELL) { |
| 2702 |
ntotal_s = 4; |
| 2703 |
} |
| 2704 |
} |
| 2705 |
|
| 2706 |
ntotal[iproc] += ntotal_s; |
| 2707 |
|
| 2708 |
proc_df_indx[iproc][ss_num[iproc] - 1][j] = indx; |
| 2709 |
proc_df_indx_cnt[iproc][ss_num[iproc] - 1][j] = 0; |
| 2710 |
|
| 2711 |
indx += ntotal_s; |
| 2712 |
|
| 2713 |
/* and set up map for sort in a minute */ |
| 2714 |
proc_df_map[iproc][j] = j; |
| 2715 |
} |
| 2716 |
|
| 2717 |
/* |
| 2718 |
* now sort the proc_df_ptr array so that it is monotonic, |
| 2719 |
* and can be searched more easily |
| 2720 |
*/ |
| 2721 |
my_sort(proc_elem_list_cnt[iproc][ss_num[iproc] - 1], |
| 2722 |
&proc_df_ptr[iproc][ss_num[iproc] - 1][0], &proc_df_map[iproc][0]); |
| 2723 |
} |
| 2724 |
|
| 2725 |
/* Allocate memory for the dfs */ |
| 2726 |
if (ntotal[iproc] > 0) { |
| 2727 |
|
| 2728 |
if (imess == 0) { |
| 2729 |
proc_ss_df[iproc][ss_num[iproc] - 1].resize(ntotal[iproc]); |
| 2730 |
} |
| 2731 |
|
| 2732 |
INT k = 0; |
| 2733 |
INT ss_indx = ss_num[iproc] - 1; |
| 2734 |
|
| 2735 |
for (size_t j = 0; j < num_elem_per_message; j++) { |
| 2736 |
|
| 2737 |
INT indx = istart_ss + j; |
| 2738 |
|
| 2739 |
/* |
| 2740 |
* Find out if this index fits in the list of this |
| 2741 |
* processors indices. |
| 2742 |
*/ |
| 2743 |
|
| 2744 |
/* 'istart_ss' <= indx <= 'istart_ss+num_elem_per_message' */ |
| 2745 |
/* 'indx' increments by 1 each time through loop */ |
| 2746 |
|
| 2747 |
/* 'proc_df_ptr[iproc][ss_indx]' is the list |
| 2748 |
and it is sorted.... |
| 2749 |
... So, don't need a binary search, can just increment |
| 2750 |
as we proceed through the loops... |
| 2751 |
*/ |
| 2752 |
if (indx >= proc_df_ptr[iproc][ss_indx][0]) { |
| 2753 |
|
| 2754 |
while (k < proc_elem_list_cnt[iproc][ss_indx] - 1 && |
| 2755 |
indx >= proc_df_ptr[iproc][ss_indx][k + 1]) { |
| 2756 |
k++; |
| 2757 |
} |
| 2758 |
|
| 2759 |
/* now get the proper location from the map */ |
| 2760 |
INT ipos_elem = proc_df_map[iproc][k]; |
| 2761 |
|
| 2762 |
INT idx_begin = proc_df_indx[iproc][ss_indx][ipos_elem]; |
| 2763 |
INT idx_end; |
| 2764 |
if ((ipos_elem + 1) < proc_elem_list_cnt[iproc][ss_indx]) { |
| 2765 |
idx_end = proc_df_indx[iproc][ss_indx][ipos_elem + 1]; |
| 2766 |
} |
| 2767 |
else { |
| 2768 |
idx_end = ntotal[iproc]; |
| 2769 |
} |
| 2770 |
|
| 2771 |
INT idx_diff = idx_end - idx_begin; |
| 2772 |
|
| 2773 |
/* |
| 2774 |
* check to see if indx really fits in this spot |
| 2775 |
* |
| 2776 |
* need to make sure that indx > the k-th value since |
| 2777 |
* bin_search_min will return 0 for anything less than |
| 2778 |
* the first entry in the list |
| 2779 |
*/ |
| 2780 |
if (indx >= proc_df_ptr[iproc][ss_indx][k] && |
| 2781 |
indx < proc_df_ptr[iproc][ss_indx][k] + idx_diff) { |
| 2782 |
|
| 2783 |
size_t df_loc = proc_df_indx[iproc][ss_indx][ipos_elem] + |
| 2784 |
proc_df_indx_cnt[iproc][ss_indx][ipos_elem]; |
| 2785 |
|
| 2786 |
proc_ss_df[iproc][ss_indx][df_loc] = ss_dist_fact[indx - istart_ss]; |
| 2787 |
|
| 2788 |
proc_ss_df_cnt[iproc][ss_indx]++; |
| 2789 |
|
| 2790 |
proc_df_indx_cnt[iproc][ss_indx][ipos_elem]++; |
| 2791 |
} |
| 2792 |
} |
| 2793 |
} /* End "for(j=0; j < num_elem_per_message; j++)" */ |
| 2794 |
} /* End "if(ntotal[iproc] > 0)" */ |
| 2795 |
} /* End "for(iproc=0; iproc <Proc_Info[2]; iproc++)" */ |
| 2796 |
} /* End "for(imess=0; imess < num_messages; imess++)" */ |
| 2797 |
} /* End "if(num_df_in_ssets[i] > 0)" */ |
| 2798 |
} /* END "if (num_elem_in_ssets[i] > 0)" */ |
| 2799 |
} /* END "for (i = 0; i < globals.Num_Side_Set; i++)" */ |
| 2800 |
|
| 2801 |
/*-------------------------------------------------------------------------*/ |
| 2802 |
/*---------------- Store Structures back into a packed form ---------------*/ |
| 2803 |
/*-------------------------------------------------------------------------*/ |
| 2804 |
|
| 2805 |
/* |
| 2806 |
* Allocate storage for permanent side set integer info vectors using one |
| 2807 |
* long malloc statement. |
| 2808 |
*/ |
| 2809 |
for (int iproc = 0; iproc < Proc_Info[2]; iproc++) { |
| 2810 |
if (globals.Num_Side_Set > 0) { |
| 2811 |
|
| 2812 |
/* |
| 2813 |
* Note that the alloc is done based on globals.Num_Side_Set, rather than |
| 2814 |
* globals.Proc_Num_Side_Sets[] due to the fact that nullptr entities are |
| 2815 |
* stored on processors not having a particular side set. |
| 2816 |
*/ |
| 2817 |
globals.Proc_SS_Ids[iproc] = |
| 2818 |
(INT *)array_alloc(__FILE__, __LINE__, 1, |
| 2819 |
(3 * globals.Num_Side_Set + 3 * globals.Proc_Num_Side_Sets[iproc] + |
| 2820 |
2 * elem_list_length[iproc] + 1), |
| 2821 |
sizeof(INT)); |
| 2822 |
globals.Proc_SS_Elem_Count[iproc] = globals.Proc_SS_Ids[iproc] + globals.Num_Side_Set; |
| 2823 |
globals.Proc_SS_Elem_Pointers[iproc] = |
| 2824 |
globals.Proc_SS_Elem_Count[iproc] + globals.Num_Side_Set; |
| 2825 |
globals.Proc_SS_DF_Count[iproc] = |
| 2826 |
globals.Proc_SS_Elem_Pointers[iproc] + globals.Proc_Num_Side_Sets[iproc]; |
| 2827 |
globals.GSide_Sets[iproc] = globals.Proc_SS_DF_Count[iproc] + globals.Num_Side_Set; |
| 2828 |
globals.Proc_SS_Elem_List[iproc] = |
| 2829 |
globals.GSide_Sets[iproc] + globals.Proc_Num_Side_Sets[iproc]; |
| 2830 |
globals.Proc_SS_Side_List[iproc] = globals.Proc_SS_Elem_List[iproc] + elem_list_length[iproc]; |
| 2831 |
globals.Proc_SS_DF_Pointers[iproc] = |
| 2832 |
globals.Proc_SS_Side_List[iproc] + elem_list_length[iproc]; |
| 2833 |
|
| 2834 |
globals.Proc_SS_GEMap_List[iproc] = |
| 2835 |
(INT *)array_alloc(__FILE__, __LINE__, 1, elem_list_length[iproc], sizeof(INT)); |
| 2836 |
} |
| 2837 |
|
| 2838 |
/* Construct the side sets global array information */ |
| 2839 |
globals.Proc_SS_Elem_List_Length[iproc] = 0; |
| 2840 |
for (int i = 0; i < globals.Proc_Num_Side_Sets[iproc]; i++) { |
| 2841 |
globals.GSide_Sets[iproc][i] = ss_proc2glob[iproc][i]; |
| 2842 |
globals.Proc_SS_Ids[iproc][i] = Side_Set_Ids[ss_proc2glob[iproc][i]]; |
| 2843 |
globals.Proc_SS_Elem_Count[iproc][i] = proc_elem_list_cnt[iproc][i]; |
| 2844 |
globals.Proc_SS_Elem_Pointers[iproc][i] = proc_elem_list_ptr[iproc][i]; |
| 2845 |
globals.Proc_SS_Elem_List_Length[iproc] += globals.Proc_SS_Elem_Count[iproc][i]; |
| 2846 |
globals.Proc_SS_DF_Count[iproc][i] = proc_ss_df_cnt[iproc][i]; |
| 2847 |
|
| 2848 |
if (i == 0) { |
| 2849 |
globals.Proc_SS_DF_Pointers[iproc][i] = 0; |
| 2850 |
} |
| 2851 |
else { |
| 2852 |
globals.Proc_SS_DF_Pointers[iproc][i] = |
| 2853 |
globals.Proc_SS_DF_Pointers[iproc][i - 1] + proc_ss_df_cnt[iproc][i - 1]; |
| 2854 |
} |
| 2855 |
} |
| 2856 |
|
| 2857 |
if (globals.Proc_Num_Side_Sets[iproc] > 0) { |
| 2858 |
int i = globals.Proc_Num_Side_Sets[iproc]; |
| 2859 |
globals.Proc_SS_DF_Pointers[iproc][i] = |
| 2860 |
globals.Proc_SS_DF_Pointers[iproc][i - 1] + proc_ss_df_cnt[iproc][i - 1]; |
| 2861 |
|
| 2862 |
if (globals.Proc_SS_DF_Pointers[iproc][i] > 0) { |
| 2863 |
globals.Proc_SS_Dist_Fact[iproc].resize(globals.Proc_SS_DF_Pointers[iproc][i]); |
| 2864 |
} |
| 2865 |
} |
| 2866 |
|
| 2867 |
/* Construct the concatenated element and side list */ |
| 2868 |
for (int i = 0; i < globals.Proc_Num_Side_Sets[iproc]; i++) { |
| 2869 |
for (INT j = 0; j < globals.Proc_SS_Elem_Count[iproc][i]; j++) { |
| 2870 |
globals.Proc_SS_Elem_List[iproc][globals.Proc_SS_Elem_Pointers[iproc][i] + j] = |
| 2871 |
proc_elem_list[iproc][i][j]; |
| 2872 |
globals.Proc_SS_Side_List[iproc][globals.Proc_SS_Elem_Pointers[iproc][i] + j] = |
| 2873 |
proc_side_list[iproc][i][j]; |
| 2874 |
globals.Proc_SS_GEMap_List[iproc][globals.Proc_SS_Elem_Pointers[iproc][i] + j] = |
| 2875 |
proc_gmap_list[iproc][i][j]; |
| 2876 |
} |
| 2877 |
for (INT j = 0; j < globals.Proc_SS_DF_Count[iproc][i]; j++) { |
| 2878 |
globals.Proc_SS_Dist_Fact[iproc][globals.Proc_SS_DF_Pointers[iproc][i] + j] = |
| 2879 |
proc_ss_df[iproc][i][j]; |
| 2880 |
} |
| 2881 |
} |
| 2882 |
} /* End "for(iproc)" */ |
| 2883 |
} /* END of routine read_side_sets */ |