Debian Codegraph @

packages/seacas/applications/nem_spread/el_exoII_io.C

raw
1 /*
2 * Copyright(C) 1999-2025 National Technology & Engineering Solutions
3 * of Sandia, LLC (NTESS). Under the terms of Contract DE-NA0003525 with
4 * NTESS, the U.S. Government retains certain rights in this software.
5 *
6 * See packages/seacas/LICENSE for details
7 */
8 #include "copy_string_cpp.h"
9 #include "el_check_monot.h" // for check_monot
10 #include "el_elm.h" // for HEXSHELL, NN_SIDE, etc
11 #include "exodusII.h" // for ex_inquire_int, etc
12 #include "fmt/ostream.h"
13 #include "nem_spread.h" // for NemSpread, second, etc
14 #include "netcdf.h" // for nc_set_fill, NC_NOFILL
15 #include "pe_common.h" // for MAX_CHUNK_SIZE
16 #include "pe_str_util_const.h" // for string_to_lower
17 #include "ps_pario_const.h" // for PIO_Time_Array, etc
18 #include "rf_allo.h" // for safe_free, array_alloc
19 #include "rf_io_const.h" // for Debug_Flag, etc
20 #include "rf_util.h" // for print_line, my_sort
21 #include "sort_utils.h" // for gds_qsort
22 #include "vector_data.h"
23 #include <cassert> // for assert
24 #include <climits> // for INT_MAX
25 #include <cstddef> // for size_t
26 #include <cstdio> // for stderr
27 #include <cstdlib> // for exit, free
28 #include <string> // for string
29 #include <vector> // for vector
30 template <typename T, typename INT> class Globals;
31
32 /*****************************************************************************/
33 /*****************************************************************************/
34 /*****************************************************************************/
35 namespace {
36 template <typename T, typename INT>
37 int check_sizes(int num_proc, std::vector<INT> &Elem_Blk_Ids, std::vector<INT> &Node_Set_Ids,
38 std::vector<INT> &Side_Set_Ids, Globals<T, INT> &globals)
39 {
40 /* Do we need 64-bit to write the entity (block/set) ids? */
41 bool ids64bit = false;
42 if (!Elem_Blk_Ids.empty()) {
43 for (int i = 0; i < globals.Num_Elem_Blk && !ids64bit; i++) {
44 if (Elem_Blk_Ids[i] >= INT_MAX) {
45 ids64bit = true;
46 }
47 }
48 }
49 if (!Node_Set_Ids.empty()) {
50 for (int i = 0; i < globals.Num_Node_Set && !ids64bit; i++) {
51 if (Node_Set_Ids[i] >= INT_MAX) {
52 ids64bit = true;
53 }
54 }
55 }
56 if (!Side_Set_Ids.empty()) {
57 for (int i = 0; i < globals.Num_Side_Set && !ids64bit; i++) {
58 if (Side_Set_Ids[i] >= INT_MAX) {
59 ids64bit = true;
60 }
61 }
62 }
63
64 /* Do we need 64-bit to write the global node/element maps? */
65 bool global_counts_64bit = false;
66 if (globals.Num_Node >= INT_MAX) {
67 global_counts_64bit = true;
68 }
69 if (globals.Num_Elem >= INT_MAX) {
70 global_counts_64bit = true;
71 }
72
73 /* Does any individual mesh have > INT_MAX nodes/elements? */
74 bool local_counts_64bit = false;
75 for (int iproc = 0; iproc < num_proc && !local_counts_64bit; iproc++) {
76 if (globals.Num_Internal_Nodes[iproc] + globals.Num_Border_Nodes[iproc] +
77 globals.Num_External_Nodes[iproc] >=
78 INT_MAX) {
79 local_counts_64bit = true;
80 }
81 if (globals.Num_Internal_Elems[iproc] + globals.Num_Border_Elems[iproc] >= INT_MAX) {
82 local_counts_64bit = true;
83 }
84 }
85
86 int mode = 0;
87 if (ids64bit) {
88 mode |= EX_IDS_INT64_DB;
89 fmt::print("-- ID output requires 64-bit integers.\n");
90 }
91
92 if (global_counts_64bit) {
93 mode |= EX_MAPS_INT64_DB;
94 fmt::print("-- Global Id map output requires 64-bit integers.\n");
95 }
96
97 if (local_counts_64bit) {
98 mode |= EX_BULK_INT64_DB;
99 fmt::print("-- Bulk data output requires 64-bit integers.\n");
100 }
101
102 return mode;
103 }
104
105 void find_message_info(size_t iunit_size, size_t max_units, size_t *num_units_per_message,
106 size_t *num_messages, size_t *num_units_left_over)
107 {
108 // Function which determines information regarding the size and number of
109 // messages to be used by the functions which broadcast the mesh information.
110
111 if (iunit_size > 0) {
112 *num_units_per_message = MAX_CHUNK_SIZE / (2 * iunit_size);
113 }
114 else {
115 fmt::print(stderr, "ERROR: find_message_info called with unit_size = 0.\n");
116 exit(1);
117 }
118
119 if (*num_units_per_message > max_units) {
120 *num_units_per_message = max_units;
121 }
122
123 *num_messages = max_units / (*num_units_per_message);
124 *num_units_left_over = max_units - *num_messages * (*num_units_per_message);
125
126 if (max_units % (*num_units_per_message) != 0) {
127 (*num_messages)++;
128 }
129 }
130 } // namespace
131
132 template void NemSpread<double, int>::load_mesh();
133 template void NemSpread<float, int>::load_mesh();
134 template void NemSpread<double, int64_t>::load_mesh();
135 template void NemSpread<float, int64_t>::load_mesh();
136
137 template <typename T, typename INT> void NemSpread<T, INT>::load_mesh()
138 {
139
140 /* Function which reads the EXODUS database which contains the mesh. It
141 * does this under the control of another EXODUS database file which
142 * contains the load-balance information. The load balance information must
143 * have been already read.
144 *
145 *----------------------------------------------------------------------------
146 *
147 * Functions called:
148 *
149 * check_exodus_error -- function which handles the error code returned by
150 * calls to EXODUS API routines.
151 * construct_lb_filename -- function which appends the string '-#' where
152 * '#' is the number of processors to the name of
153 * the EXODUS filename.
154 * read_lb -- function which reads the load-balance information
155 * from an EXODUS database for a given processor.
156 * read_coord -- function which reads the nodal coordinates information
157 * from an EXODUS database for a given processor.
158 * read_elem_blk_ids-- Function which read the element block ids and other
159 * global information specific to the element block level.
160 * read_elem_blk -- Function which reads the element level information
161 * for each element block in the mesh
162 * read_node_sets -- function which reads the node sets information from an
163 * EXODUS database for a given processor.
164 * read_side_sets -- function which reads the side sets information from an
165 * EXODUS database for a given processor.
166 *
167 *----------------------------------------------------------------------------
168 */
169
170 /* Local variables */
171
172 int mesh_exoid = 0;
173 int cpu_ws;
174 float version;
175 double start_time = 0.0;
176
177 /* Allocate some memory for each processor read by this processor */
178 globals.Proc_Num_Elem_Blk = (int *)array_alloc(__FILE__, __LINE__, 1, Proc_Info[2], sizeof(int));
179 globals.Proc_Num_Node_Sets = (int *)array_alloc(__FILE__, __LINE__, 1, Proc_Info[2], sizeof(int));
180 globals.Proc_Num_Side_Sets = (int *)array_alloc(__FILE__, __LINE__, 1, Proc_Info[2], sizeof(int));
181
182 globals.Proc_NS_List_Length =
183 (INT *)array_alloc(__FILE__, __LINE__, 1, Proc_Info[2], sizeof(INT));
184 globals.Proc_SS_Elem_List_Length =
185 (INT *)array_alloc(__FILE__, __LINE__, 1, Proc_Info[2], sizeof(INT));
186
187 /* Initialize */
188 for (int iproc = 0; iproc < Proc_Info[2]; iproc++) {
189 globals.Proc_Num_Elem_Blk[iproc] = 0;
190 globals.Proc_Num_Node_Sets[iproc] = 0;
191 globals.Proc_Num_Side_Sets[iproc] = 0;
192 globals.Proc_NS_List_Length[iproc] = 0;
193 globals.Proc_SS_Elem_List_Length[iproc] = 0;
194 }
195
196 globals.GElem_Blks = (INT **)array_alloc(__FILE__, __LINE__, 1, 24 * Proc_Info[2], sizeof(INT *));
197 globals.Proc_Nodes_Per_Elem = globals.GElem_Blks + Proc_Info[2];
198 globals.Proc_Elem_Blk_Ids = globals.Proc_Nodes_Per_Elem + Proc_Info[2];
199 globals.Proc_Elem_Blk_Types = globals.Proc_Elem_Blk_Ids + Proc_Info[2];
200 globals.Proc_Num_Attr = globals.Proc_Elem_Blk_Types + Proc_Info[2];
201 globals.Proc_Num_Elem_In_Blk = globals.Proc_Num_Attr + Proc_Info[2];
202 globals.Proc_Connect_Ptr = globals.Proc_Num_Elem_In_Blk + Proc_Info[2];
203 globals.Proc_Elem_Connect = globals.Proc_Connect_Ptr + Proc_Info[2];
204 globals.Proc_NS_Ids = globals.Proc_Elem_Connect + Proc_Info[2];
205 globals.Proc_NS_Count = globals.Proc_NS_Ids + Proc_Info[2];
206 globals.Proc_NS_DF_Count = globals.Proc_NS_Count + Proc_Info[2];
207 globals.Proc_NS_Pointers = globals.Proc_NS_DF_Count + Proc_Info[2];
208 globals.Proc_NS_List = globals.Proc_NS_Pointers + Proc_Info[2];
209 globals.GNode_Sets = globals.Proc_NS_List + Proc_Info[2];
210 globals.Proc_NS_GNMap_List = globals.GNode_Sets + Proc_Info[2];
211 globals.Proc_SS_Ids = globals.Proc_NS_GNMap_List + Proc_Info[2];
212 globals.Proc_SS_Elem_Count = globals.Proc_SS_Ids + Proc_Info[2];
213 globals.Proc_SS_DF_Count = globals.Proc_SS_Elem_Count + Proc_Info[2];
214 globals.Proc_SS_Elem_Pointers = globals.Proc_SS_DF_Count + Proc_Info[2];
215 globals.Proc_SS_Elem_List = globals.Proc_SS_Elem_Pointers + Proc_Info[2];
216 globals.Proc_SS_Side_List = globals.Proc_SS_Elem_List + Proc_Info[2];
217 globals.Proc_SS_DF_Pointers = globals.Proc_SS_Side_List + Proc_Info[2];
218 globals.GSide_Sets = globals.Proc_SS_DF_Pointers + Proc_Info[2];
219 globals.Proc_SS_GEMap_List = globals.GSide_Sets + Proc_Info[2];
220
221 globals.Proc_Global_Elem_Id_Map.resize(Proc_Info[2]);
222 globals.Proc_Global_Node_Id_Map.resize(Proc_Info[2]);
223
224 /* Initialize */
225 for (int iproc = 0; iproc < 24 * Proc_Info[2]; iproc++) {
226 globals.GElem_Blks[iproc] = nullptr;
227 }
228
229 globals.Coor = (T ***)array_alloc(__FILE__, __LINE__, 1, Proc_Info[2], sizeof(T **));
230
231 /* Initialize */
232 for (int iproc = 0; iproc < Proc_Info[2]; iproc++) {
233 globals.Coor[iproc] = nullptr;
234 }
235
236 globals.Proc_Elem_Attr.resize(Proc_Info[2]);
237 globals.Proc_NS_Dist_Fact.resize(Proc_Info[2]);
238 globals.Proc_SS_Dist_Fact.resize(Proc_Info[2]);
239
240 /* Check for a problem which has too many processors for a given mesh */
241
242 if (globals.Num_Node / Proc_Info[0] < 1) {
243 fmt::print(stderr,
244 "[{}]: ERROR: Problem divided among too many "
245 "processors.\n",
246 __func__);
247 exit(1);
248 }
249 else if (globals.Num_Elem / Proc_Info[0] < 1) {
250 fmt::print(stderr,
251 "[{}]: ERROR: Problem divided among too many "
252 "processors.\n",
253 __func__);
254 exit(1);
255 }
256
257 /* Open the EXODUS mesh file */
258
259 /* computing precision should be the same as the database precision
260 *
261 * EXCEPTION: if the io_ws is smaller than the machine precision,
262 * ie - database with io_ws == 4 on a Cray (sizeof(float) == 8),
263 * then the cpu_ws must be the machine precision.
264 */
265 {
266 cpu_ws = io_ws;
267 int mode = EX_READ | int64api;
268 mesh_exoid = ex_open(ExoFile.c_str(), mode, &cpu_ws, &io_ws, &version);
269 if (mesh_exoid < 0) {
270 fmt::print(stderr, "{}Exodus returned error opening mesh file, {}\n", __func__, ExoFile);
271 exit(1);
272 }
273 }
274 mesh_exoid += selected_change_set;
275
276 auto max_name_length = ex_inquire_int(mesh_exoid, EX_INQ_DB_MAX_USED_NAME_LENGTH);
277 ex_set_max_name_length(mesh_exoid, max_name_length);
278
279 globals.Num_QA_Recs = ex_inquire_int(mesh_exoid, EX_INQ_QA);
280
281 /* Add a QA record for the spreader */
282 globals.Num_QA_Recs++;
283
284 int length_qa = 4 * (globals.Num_QA_Recs);
285 globals.QA_Record =
286 reinterpret_cast<char **>(array_alloc(__FILE__, __LINE__, 1, length_qa, sizeof(char *)));
287 for (int i1 = 0, index = 0; i1 < globals.Num_QA_Recs; i1++) {
288 for (int i2 = 0; i2 < 4; i2++) {
289 globals.QA_Record[index++] = reinterpret_cast<char *>(
290 array_alloc(__FILE__, __LINE__, 1, MAX_STR_LENGTH + 1, sizeof(char)));
291 }
292 }
293
294 if (ex_get_qa(mesh_exoid, (char *(*)[4])globals.QA_Record) < 0) {
295 fmt::print(stderr, "[{}]: ERROR, could not get QA record(s)\n", __func__);
296 exit(1);
297 }
298
299 /* Read in the information records */
300 globals.Num_Info_Recs = ex_inquire_int(mesh_exoid, EX_INQ_INFO);
301
302 if (globals.Num_Info_Recs > 0) {
303 /* Allocate the Information records */
304 globals.Info_Record = (char **)array_alloc(__FILE__, __LINE__, 2, globals.Num_Info_Recs,
305 MAX_LINE_LENGTH + 1, sizeof(char));
306 if (!globals.Info_Record) {
307 fmt::print(stderr, "[{}]: ERROR, insufficient memory!\n", __func__);
308 exit(1);
309 }
310
311 if (ex_get_info(mesh_exoid, globals.Info_Record) < 0) {
312 fmt::print(stderr, "[{}]: ERROR, could not get Info record(s)\n", __func__);
313 exit(1);
314 }
315 }
316
317 /* Read in the assembly information */
318 {
319 ex_init_params ex_info;
320 ex_get_init_ext(mesh_exoid, &ex_info);
321 globals.Num_Assemblies = ex_info.num_assembly;
322 }
323
324 if (globals.Num_Assemblies > 0) {
325 globals.Assemblies.resize(globals.Num_Assemblies);
326 for (auto &assembly : globals.Assemblies) {
327 assembly.name = nullptr;
328 assembly.entity_list = nullptr;
329 }
330 ex_get_assemblies(mesh_exoid, Data(globals.Assemblies));
331
332 for (auto &assembly : globals.Assemblies) {
333 assembly.entity_list = new int64_t[assembly.entity_count];
334 }
335
336 // Now get the assembly entity lists...
337 ex_get_assemblies(mesh_exoid, Data(globals.Assemblies));
338 }
339
340 /* Read in the coordinate frame information */
341 globals.Num_Coordinate_Frames = ex_inquire_int(mesh_exoid, EX_INQ_COORD_FRAMES);
342
343 if (globals.Num_Coordinate_Frames > 0) {
344
345 /* Allocate the Coordinate Frame records */
346 globals.Coordinate_Frame_Ids =
347 (INT *)array_alloc(__FILE__, __LINE__, 1, globals.Num_Coordinate_Frames, sizeof(INT));
348 if (!globals.Coordinate_Frame_Ids) {
349 fmt::print(stderr, "[{}]: ERROR, insufficient memory!\n", __func__);
350 exit(1);
351 }
352
353 globals.Coordinate_Frame_Coordinates =
354 (T *)array_alloc(__FILE__, __LINE__, 1, 9 * globals.Num_Coordinate_Frames, sizeof(T));
355 if (!globals.Coordinate_Frame_Coordinates) {
356 fmt::print(stderr, "[{}]: ERROR, insufficient memory!\n", __func__);
357 exit(1);
358 }
359
360 globals.Coordinate_Frame_Tags =
361 (char *)array_alloc(__FILE__, __LINE__, 1, globals.Num_Coordinate_Frames, sizeof(char));
362 if (!globals.Coordinate_Frame_Tags) {
363 fmt::print(stderr, "[{}]: ERROR, insufficient memory!\n", __func__);
364 exit(1);
365 }
366
367 int num_frames = 0;
368 if (ex_get_coordinate_frames(mesh_exoid, &num_frames, globals.Coordinate_Frame_Ids,
369 globals.Coordinate_Frame_Coordinates,
370 globals.Coordinate_Frame_Tags) < 0) {
371 fmt::print(stderr, "[{}]: ERROR, could not get Coordinate Frame record(s)\n", __func__);
372 exit(1);
373 }
374 if (num_frames != globals.Num_Coordinate_Frames) {
375 fmt::print(stderr, "[{}]: ERROR, frame count inconsistency\n", __func__);
376 exit(1);
377 }
378 }
379
380 /*
381 * Allocate Temporary Arrays that are only used in el_exoII_io.c (Note: Calls
382 * to array_alloc are minimized by using pointer arithmetic. Resulting memory
383 * structure may be assumed during later broadcast routines)
384 */
385
386 if (globals.Num_Elem_Blk <= 0) {
387 fmt::print(stderr, "ERROR, globals.Num_Elem_Blk = {}\n", globals.Num_Elem_Blk);
388 exit(1);
389 }
390
391 Num_Elem_In_Blk.resize(globals.Num_Elem_Blk);
392 Num_Nodes_Per_Elem.resize(globals.Num_Elem_Blk);
393 Num_Attr_Per_Elem.resize(globals.Num_Elem_Blk);
394 Elem_Blk_Ids.resize(globals.Num_Elem_Blk);
395
396 Elem_Blk_Types = (char **)array_alloc(__FILE__, __LINE__, 2, globals.Num_Elem_Blk,
397 MAX_STR_LENGTH + 1, sizeof(char));
398 Elem_Blk_Names = (char **)array_alloc(__FILE__, __LINE__, 2, globals.Num_Elem_Blk,
399 max_name_length + 1, sizeof(char));
400 Elem_Blk_Attr_Names =
401 (char ***)array_alloc(__FILE__, __LINE__, 1, globals.Num_Elem_Blk, sizeof(char **));
402
403 Node_Set_Ids.resize(globals.Num_Node_Set);
404 std::vector<INT> num_nodes_in_node_set(globals.Num_Node_Set);
405 std::vector<INT> num_df_in_nsets(globals.Num_Node_Set);
406
407 if (globals.Num_Node_Set > 0) {
408 Node_Set_Names = (char **)array_alloc(__FILE__, __LINE__, 2, globals.Num_Node_Set,
409 max_name_length + 1, sizeof(char));
410 }
411
412 Side_Set_Ids.resize(globals.Num_Side_Set);
413 std::vector<INT> num_elem_in_ssets(globals.Num_Side_Set);
414 std::vector<INT> num_df_in_ssets(globals.Num_Side_Set);
415
416 if (globals.Num_Side_Set > 0) {
417 Side_Set_Names = (char **)array_alloc(__FILE__, __LINE__, 2, globals.Num_Side_Set,
418 max_name_length + 1, sizeof(char));
419 }
420
421 /*
422 * Process the Global Element Block IDs and associated information related to
423 * element blocks (i.e., read them, broadcast them, and check them against
424 * the input file).
425 */
426 start_time = second();
427
428 read_elem_blk_ids(mesh_exoid, max_name_length);
429
430 fmt::print("\tTime to read element block IDs: {:.4f}\n", second() - start_time);
431
432 /*
433 * Process the Node Set IDs and associated information related to node sets
434 * (i.e., read them, broadcast them, and check them against the input file).
435 */
436
437 if (globals.Num_Node_Set > 0) {
438 start_time = second();
439 read_node_set_ids(mesh_exoid, num_nodes_in_node_set, num_df_in_nsets, max_name_length);
440
441 fmt::print("\tTime to read node set IDs: {:.4f}\n", second() - start_time);
442 }
443
444 /*
445 * Process the Side Set IDs and associated information related to side sets
446 * (i.e., read them, broadcast them, and check them against the input file).
447 */
448 if (globals.Num_Side_Set > 0) {
449 start_time = second();
450 read_side_set_ids(mesh_exoid, num_elem_in_ssets, num_df_in_ssets, max_name_length);
451 fmt::print("\tTime to read side set IDs: {:.4f}\n", second() - start_time);
452 }
453
454 /*
455 * Process the element block information. Find out which element blocks have
456 * elements needed by which processors. Set-up and fill in vectors which
457 * have length equal to the number of element blocks on the processor,
458 * globals.Num_Elem_Blk.
459 */
460 start_time = second();
461
462 extract_elem_blk();
463
464 fmt::print("\tTime to extract element block information: {:.4f}\n", second() - start_time);
465
466 /*
467 * Read the mesh information from the exodus file and broadcast it to the
468 * processors. NOTE: this will only read as much information as will fit in
469 * the communication buffer. It will then send the information to all the
470 * processors so they can extract the information they need. Then, the next
471 * block of mesh data will be read and sent, etc., until the entire mesh has
472 * been read and distributed to the processors. Note, that the load balance
473 * information has already been distributed by this point. Therefore, each
474 * node already nodes which global nodes it owns, which elements it has
475 * global nodes in that it owns, and which ghost nodes it needs information
476 * about.
477 *
478 * The following information is defined as the mesh information:
479 *
480 * 1 - Coordinate values of global nodes
481 * 2 - Element connectivity graph
482 * 3 - Attributes of the elements
483 * 4 - Composition of Node Sets
484 * 5 - Composition of Side Sets
485 */
486
487 /* Read the coordinates, broadcast, and sort */
488
489 start_time = second();
490
491 read_coord(mesh_exoid, max_name_length);
492
493 fmt::print("\tTime to read nodal coordinates: {:.4f}\n", second() - start_time);
494
495 /*
496 * Read the element connectivity and attributes information. Broadcast it
497 * and extract only the information that the current processor needs.
498 */
499 start_time = second();
500 read_elem_blk(mesh_exoid);
501 fmt::print("\tTime to read element blocks: {:.4f}\n", second() - start_time);
502
503 /* Read the node sets and sort */
504 if (globals.Num_Node_Set > 0) {
505 start_time = second();
506 read_node_sets(mesh_exoid, num_nodes_in_node_set, num_df_in_nsets);
507 fmt::print("\tTime to read node sets: {:.4f}\n", second() - start_time);
508 }
509
510 /* Assign the element types. */
511 if (globals.Num_Side_Set > 0) {
512 start_time = second();
513 create_elem_types(); /* globals.Elem_Type only used in sideset read */
514 fmt::print("\tTime to categorize element types: {:.4f}\n", second() - start_time);
515
516 /* Read the side sets and sort */
517 start_time = second();
518 read_side_sets(mesh_exoid, num_elem_in_ssets, num_df_in_ssets);
519 fmt::print("\tTime to read side sets: {:.4f}\n", second() - start_time);
520 }
521
522 /* Close the EXODUS mesh file */
523 check_exodus_error(ex_close(mesh_exoid), "ex_close");
524
525 /************* Output the information to the parallel disks *****************/
526 /*==========================================================================*/
527 /*==========================================================================*/
528
529 /* Generate the processor to disk map */
530 gen_disk_map(&PIO_Info, Proc_Info, 0, 1);
531
532 /* Generate the parallel exodus file name */
533 /* See if any '/' in the name. IF present, isolate the basename of the file */
534 std::string cTemp;
535
536 size_t found = Output_File_Base_Name.find_last_of('/');
537 if (found != std::string::npos) {
538 /* There is a path separator. Get the portion after the
539 * separator
540 */
541 cTemp = Output_File_Base_Name.substr(found + 1);
542 }
543 else {
544 /* No separator; this is already just the basename... */
545 cTemp = Output_File_Base_Name;
546 }
547
548 if (PIO_Info.Exo_Extension.empty()) {
549 cTemp += ".par";
550 }
551 else {
552 cTemp += PIO_Info.Exo_Extension;
553 }
554
555 /* Check sizes to see if need to store using 64-bit on the database */
556 int db_mode = check_sizes(Proc_Info[2], Elem_Blk_Ids, Node_Set_Ids, Side_Set_Ids, globals);
557 if (force64db) {
558 fmt::print("-- Command-line option forcing output to use all 64-bit integers.\n");
559 db_mode |= EX_ALL_INT64_DB;
560 }
561
562 for (int iproc = Proc_Info[4]; iproc < Proc_Info[4] + Proc_Info[5]; iproc++) {
563 std::string Parallel_File_Name = gen_par_filename(cTemp, Proc_Ids[iproc], Proc_Info[0]);
564
565 /* Create the parallel Exodus file for writing */
566 if (Debug_Flag >= 7) {
567 fmt::print("{}Parallel mesh file name is {}\n", __func__, Parallel_File_Name);
568 }
569 else {
570 if (iproc % 10 == 0 || iproc == Proc_Info[2] - 1) {
571 fmt::print("{}", iproc);
572 }
573 else {
574 fmt::print(".");
575 }
576 }
577
578 int mode = EX_CLOBBER;
579 mode |= int64api;
580 mode |= db_mode;
581 if ((mesh_exoid = ex_create(Parallel_File_Name.c_str(), mode, &cpu_ws, &io_ws)) == -1) {
582
583 fmt::print(stderr, "[{}] Could not create parallel Exodus file:\n\t{}\n", __func__,
584 Parallel_File_Name);
585 exit(1);
586 }
587
588 ex_set_max_name_length(mesh_exoid, max_name_length);
589
590 /* Set fill mode off... */
591 {
592 int old_fill;
593 nc_set_fill(mesh_exoid, NC_NOFILL, &old_fill);
594 }
595
596 if (Debug_Flag >= 7) {
597 fmt::print("{}Parallel mesh file id is {}\n", __func__, mesh_exoid);
598 }
599
600 /* Write out a parallel mesh file local to each processor */
601 write_parExo_data(mesh_exoid, max_name_length, iproc, num_nodes_in_node_set, num_elem_in_ssets,
602 Num_Elem_In_Blk);
603
604 /* Close the parallel exodus file */
605 if (ex_close(mesh_exoid) == -1) {
606 fmt::print(stderr, "{}Could not close the parallel Exodus file\n", __func__);
607 exit(1);
608 }
609 } /* End "for(iproc=0; iproc <Proc_Info[2]; iproc++)" */
610
611 if (Debug_Flag >= 4) {
612 fmt::print("\n\n\t\tTIMING TABLE FOR PROCESSORS\n");
613 fmt::print("===========================================================\n");
614 fmt::print("[0]: {:6.2f}\n", PIO_Time_Array[0]);
615 for (int i1 = 1; i1 < 25; i1++) {
616 fmt::print("\t\t[{}]: {:6.2f}\n", i1, PIO_Time_Array[i1]);
617 }
618 fmt::print("\nOutput rate: {:6.2f} kB/s\n", PIO_Time_Array[25]);
619 }
620 else if (Debug_Flag >= 1) {
621 fmt::print("\n\nOutput rate: {:6.2f} kB/s\n", PIO_Time_Array[25]);
622 }
623
624 /*--------------------------------------------------------------------------*/
625 /*--------------------------------------------------------------------------*/
626
627 /* Free QA records */
628 for (int i1 = 0; i1 < 4 * (globals.Num_QA_Recs); i1++) {
629 safe_free((void **)&(globals.QA_Record[i1]));
630 }
631
632 safe_free((void **)&globals.QA_Record);
633
634 for (int i1 = 0; i1 < globals.Num_Elem_Blk; i1++) {
635 if (Num_Attr_Per_Elem[i1] > 0) {
636 safe_free((void **)&(Elem_Blk_Attr_Names[i1]));
637 }
638 }
639 safe_free((void **)&Elem_Blk_Attr_Names);
640
641 /* Free Coordinate Frames */
642 if (globals.Num_Coordinate_Frames > 0) {
643 safe_free((void **)&globals.Coordinate_Frame_Ids);
644 safe_free((void **)&globals.Coordinate_Frame_Coordinates);
645 safe_free((void **)&globals.Coordinate_Frame_Tags);
646 }
647
648 // Clear out allocated memory for assembly entity_list...
649 for (auto &assembly : globals.Assemblies) {
650 free(assembly.name);
651 delete[] assembly.entity_list;
652 assembly.name = nullptr;
653 assembly.entity_list = nullptr;
654 }
655
656 /* done with the Coordinate names */
657 for (int i1 = 0; i1 < globals.Num_Dim; i1++) {
658 safe_free((void **)&(Coord_Name[i1]));
659 }
660
661 /* Free some local arrays */
662 Num_Elem_In_Blk.clear();
663 Node_Set_Ids.clear();
664 Side_Set_Ids.clear();
665
666 safe_free((void **)&Elem_Blk_Types);
667 safe_free((void **)&Node_Set_Names);
668 safe_free((void **)&Side_Set_Names);
669 safe_free((void **)&Elem_Blk_Names);
670
671 safe_free((void **)&Restart_Info.NV_Name);
672 safe_free((void **)&Restart_Info.EV_Name);
673 safe_free((void **)&Restart_Info.GV_Name);
674 safe_free((void **)&Restart_Info.NSV_Name);
675 safe_free((void **)&Restart_Info.SSV_Name);
676
677 /*
678 * free up some other memory so that there is more
679 * memory available for reading restart variables
680 */
681 for (int iproc = 0; iproc < Proc_Info[2]; iproc++) {
682 safe_free((void **)&(globals.Proc_Connect_Ptr[iproc]));
683 safe_free((void **)&(globals.Proc_Elem_Connect[iproc]));
684
685 safe_free((void **)&(globals.Coor[iproc]));
686 globals.Proc_Elem_Attr[iproc].clear();
687 globals.Proc_NS_Dist_Fact[iproc].clear();
688 globals.Proc_SS_Dist_Fact[iproc].clear();
689 }
690 globals.N_Comm_Map.clear();
691 globals.E_Comm_Map.clear();
692 globals.Proc_Elem_Attr.clear();
693 globals.Proc_NS_Dist_Fact.clear();
694 globals.Proc_SS_Dist_Fact.clear();
695 fmt::print("\n");
696
697 } /* END of routine load_mesh () *********************************************/
698
699 /*****************************************************************************/
700 /*****************************************************************************/
701 /*****************************************************************************/
702
703 template <typename T, typename INT>
704 void NemSpread<T, INT>::read_elem_blk_ids(int mesh_exoid, int max_name_length)
705 {
706
707 /* This function reads part of the element block info from the EXODUS file.
708 * It reads all information having a length equal to the number of elements
709 * blocks, specifically.
710 *
711 * The function then broadcasts this information to all processors.
712 *
713 * The function also calls check_matrl_blks, which checks the element block ids
714 * in the input file and in the mesh file for consistency (and prints out a
715 * table for a large enough value of Debug_Flag).
716 */
717
718 /* Get the Element Block IDs from the input file */
719 check_exodus_error(ex_get_ids(mesh_exoid, EX_ELEM_BLOCK, Data(Elem_Blk_Ids)), "ex_get_ids");
720
721 check_exodus_error(ex_get_names(mesh_exoid, EX_ELEM_BLOCK, Elem_Blk_Names), "ex_get_names");
722
723 /*
724 * Get from the input file:
725 * Number of Elements in each element block
726 * Number of nodes per element in each element block
727 * Number of attributes per element in each element block
728 * The element type for elements in each element block
729 */
730 for (int i = 0; i < globals.Num_Elem_Blk; i++) {
731
732 check_exodus_error(ex_get_block(mesh_exoid, EX_ELEM_BLOCK, Elem_Blk_Ids[i], Elem_Blk_Types[i],
733 &Num_Elem_In_Blk[i], &Num_Nodes_Per_Elem[i], nullptr, nullptr,
734 &Num_Attr_Per_Elem[i]),
735 "ex_get_elem_block");
736
737 /* Convert element block types to lower case here */
738 string_to_lower(Elem_Blk_Types[i], '\0');
739
740 /* Allocate space for attribute names (if any), read and store. */
741 if (Num_Attr_Per_Elem[i] > 0) {
742 Elem_Blk_Attr_Names[i] = (char **)array_alloc(__FILE__, __LINE__, 2, Num_Attr_Per_Elem[i],
743 max_name_length + 1, sizeof(char));
744 check_exodus_error(
745 ex_get_attr_names(mesh_exoid, EX_ELEM_BLOCK, Elem_Blk_Ids[i], Elem_Blk_Attr_Names[i]),
746 "ex_get_attr_names");
747 }
748 else {
749 Elem_Blk_Attr_Names[i] = nullptr;
750 }
751 }
752 }
753
754 /*****************************************************************************/
755 /*****************************************************************************/
756 /*****************************************************************************/
757
758 template <typename T, typename INT>
759 void NemSpread<T, INT>::read_node_set_ids(int mesh_exoid, std::vector<INT> &num_nodes_in_node_set,
760 std::vector<INT> &num_df_in_nsets,
761 int /*max_name_length*/)
762
763 /*
764 * This function reads part of the node set info from the EXODUS file. It
765 * reads all information having a length equal to the number of node sets,
766 * specifically.
767 *
768 * The function then broadcasts this information to all processors.
769 *
770 */
771
772 {
773 if (globals.Num_Node_Set > 0) {
774 int error = ex_get_ids(mesh_exoid, EX_NODE_SET, Data(Node_Set_Ids));
775 check_exodus_error(error, "ex_get_node_set_ids");
776
777 error = ex_get_names(mesh_exoid, EX_NODE_SET, Node_Set_Names);
778 check_exodus_error(error, "ex_get_node_set_ids");
779
780 for (int i = 0; i < globals.Num_Node_Set; i++) {
781 check_exodus_error(ex_get_set_param(mesh_exoid, EX_NODE_SET, Node_Set_Ids[i],
782 &num_nodes_in_node_set[i], &num_df_in_nsets[i]),
783 "ex_get_set_param");
784 }
785 }
786
787 /* Output debug info */
788 if ((Debug_Flag > 1)) {
789 fmt::print("\n\n");
790 print_line("=", 79);
791 fmt::print("\tTABLE OF NODE SET ID\'s\n\n");
792 fmt::print("Node_Set_Num ID globals.Num_Nodes\n");
793 print_line("-", 79);
794
795 if (globals.Num_Node_Set > 0) {
796 for (int i = 0; i < globals.Num_Node_Set; i++) {
797 fmt::print("{:6d}{:11d}{:12d}\n", i, Node_Set_Ids[i], num_nodes_in_node_set[i]);
798 }
799 }
800
801 else {
802 fmt::print("\tNO NODE SETS ARE DEFINED IN THE MESH FILE\n");
803 }
804 print_line("=", 79);
805 fmt::print("\n");
806 }
807 }
808
809 /*****************************************************************************/
810 /*****************************************************************************/
811 /*****************************************************************************/
812
813 template <typename T, typename INT>
814 void NemSpread<T, INT>::read_side_set_ids(int mesh_exoid, std::vector<INT> &num_elem_in_ssets,
815 std::vector<INT> &num_df_in_ssets,
816 int /*max_name_length*/)
817
818 /* This function reads part of the side set info from the EXODUS file. It
819 * reads all information having a length equal to the number of side sets,
820 * specifically.
821 *
822 * The function then broadcasts this information to all processors.
823 *
824 */
825
826 {
827 if (globals.Num_Side_Set > 0) {
828 int error = ex_get_ids(mesh_exoid, EX_SIDE_SET, Data(Side_Set_Ids));
829 check_exodus_error(error, "ex_get_side_set_ids");
830
831 error = ex_get_names(mesh_exoid, EX_SIDE_SET, Side_Set_Names);
832 check_exodus_error(error, "ex_get_side_set_ids");
833
834 for (int i = 0; i < globals.Num_Side_Set; i++) {
835 check_exodus_error(ex_get_set_param(mesh_exoid, EX_SIDE_SET, Side_Set_Ids[i],
836 &num_elem_in_ssets[i], &num_df_in_ssets[i]),
837 "ex_get_set_param");
838 }
839 }
840
841 /* Output debug information */
842 if (Debug_Flag > 1) {
843 fmt::print("\n\n");
844 print_line("=", 79);
845 fmt::print("\tTABLE OF SIDE SET ID\'s\n\n");
846 fmt::print("Side_Set_Num ID Number Elements\n");
847 print_line("-", 79);
848
849 if (globals.Num_Side_Set > 0) {
850 for (int i = 0; i < globals.Num_Side_Set; i++) {
851 fmt::print("{:6d}{:11d} {:12}\n", i, Side_Set_Ids[i],
852 fmt::group_digits(num_elem_in_ssets[i]));
853 }
854 }
855
856 else {
857 fmt::print("\tNO SIDE SETS ARE DEFINED IN THE MESH FILE\n");
858 }
859 print_line("=", 79);
860 fmt::print("\n");
861 }
862 }
863
864 /*****************************************************************************/
865 /*****************************************************************************/
866 /*****************************************************************************/
867
868 template <typename T, typename INT>
869 void NemSpread<T, INT>::read_coord(int exoid, int max_name_length)
870 {
871
872 /* Function which reads the nodal coordinates information from an
873 * EXODUS database for a given processor.
874 */
875
876 /*
877 * Calculate the size of the coordinate space for each processors coordinate matrix.
878 */
879 for (int iproc = Proc_Info[4]; iproc < Proc_Info[4] + Proc_Info[5]; iproc++) {
880 size_t itotal_nodes = globals.Num_Internal_Nodes[iproc] + globals.Num_Border_Nodes[iproc] +
881 globals.Num_External_Nodes[iproc];
882
883 /* Allocate permanent storage for the coordinates */
884 if (itotal_nodes > 0) {
885 globals.Coor[iproc] =
886 (T **)array_alloc(__FILE__, __LINE__, 2, globals.Num_Dim, itotal_nodes, sizeof(T));
887 }
888 else {
889 globals.Coor[iproc] = nullptr;
890 }
891 }
892
893 /* Allocate temporary space to hold 1 dimensions worth of coordinates... */
894 std::vector<T> coord(globals.Num_Node);
895
896 /* Read in the coordinates and broadcast to the processors */
897 for (int idim = 0; idim < globals.Num_Dim; idim++) {
898 switch (idim) {
899 case 0:
900 check_exodus_error(ex_get_coord(exoid, Data(coord), nullptr, nullptr), "ex_get_coord");
901 break;
902 case 1:
903 check_exodus_error(ex_get_coord(exoid, nullptr, Data(coord), nullptr), "ex_get_coord");
904 break;
905 case 2:
906 check_exodus_error(ex_get_coord(exoid, nullptr, nullptr, Data(coord)), "ex_get_coord");
907 break;
908 }
909
910 for (int iproc = Proc_Info[4]; iproc < Proc_Info[4] + Proc_Info[5]; iproc++) {
911 size_t itotal_nodes = globals.Num_Internal_Nodes[iproc] + globals.Num_Border_Nodes[iproc] +
912 globals.Num_External_Nodes[iproc];
913
914 for (size_t j = 0; j < itotal_nodes; j++) {
915 size_t inode = globals.GNodes[iproc][j];
916 globals.Coor[iproc][idim][j] = coord[inode];
917 }
918 }
919 }
920 coord.clear();
921
922 for (int i = 0; i < globals.Num_Dim; i++) {
923 Coord_Name[i] = reinterpret_cast<char *>(
924 array_alloc(__FILE__, __LINE__, 1, max_name_length + 1, sizeof(char)));
925 }
926
927 /* Get the coordinate names */
928 if (ex_get_coord_names(exoid, Coord_Name) < 0) {
929 fmt::print(stderr, "ERROR:Unable to obtain coordinate names\n");
930 exit(1);
931 }
932
933 /* Handle global node ids... */
934 std::vector<INT> global_node_ids(globals.Num_Node);
935
936 check_exodus_error(ex_get_id_map(exoid, EX_NODE_MAP, Data(global_node_ids)), "ex_get_id_map");
937 /*
938 * Check whether map is sequential (1..globals.Num_Node). If it is, then it
939 * provides no information and we don't need to store it in the
940 * output databases.
941 */
942 {
943 bool sequential = true;
944 for (size_t i = 0; i < globals.Num_Node; i++) {
945 if ((size_t)global_node_ids[i] != i + 1) {
946 sequential = false;
947 break;
948 }
949 }
950
951 // Check that map is valid 1 <= global_node_id[*]
952 // If not, output a warning and disable the map.
953 for (size_t i = 0; i < globals.Num_Node; i++) {
954 if (global_node_ids[i] <= 0) {
955 fmt::print(stderr,
956 "---------------------------------------------------------------------\n"
957 "ERROR: Local node {} has a global id of {} which is invalid.\n"
958 " All global ids must be greater than 0. The map will be ignored.\n"
959 "---------------------------------------------------------------------\n",
960 fmt::group_digits(i + 1), fmt::group_digits(global_node_ids[i]));
961 sequential = true; // Map is invalid, ignore it.
962 break;
963 }
964 }
965
966 if (!sequential) {
967 for (int iproc = Proc_Info[4]; iproc < Proc_Info[4] + Proc_Info[5]; iproc++) {
968
969 size_t itotal_nodes = globals.Num_Internal_Nodes[iproc] + globals.Num_Border_Nodes[iproc] +
970 globals.Num_External_Nodes[iproc];
971
972 globals.Proc_Global_Node_Id_Map[iproc].resize(itotal_nodes);
973 extract_global_node_ids(global_node_ids, globals.Num_Node, iproc);
974 }
975 }
976 }
977 } /* END of routine read_coord */
978
979 /*****************************************************************************/
980 /*****************************************************************************/
981 /*****************************************************************************/
982
983 template <typename T, typename INT> void NemSpread<T, INT>::extract_elem_blk()
984
985 /* Function which calculates the element block information for the current
986 * processor, given the global element block information. If an error is
987 * tripped, the program will set a flag, and abort at the end of the routine.
988 *
989 * The following global variables are allocated and calculated in this routine:
990 *
991 * globals.Num_Elem_Blk = Number of element blocks on the current
992 * processor
993 * globals.GElem_Blks [globals.Num_Elem_Blk]
994 * = Map from the local element block number
995 * to the global element block number.
996 * globals.Proc_Nodes_Per_Elem [globals.Num_Elem_Blk]
997 * = Number of nodes per element for each
998 * block on the current processor
999 * globals.Proc_Elem_Blk_Ids [globals.Num_Elem_Blk]
1000 * = Element block id's for the processor's
1001 * element blocks
1002 * globals.Proc_Elem_Blk_Types [globals.Num_Elem_Blk]
1003 * = Element block types for the processor's
1004 * element blocks
1005 * (this is a unique integer number)
1006 * globals.Proc_Num_Attr [globals.Num_Elem_Blk]
1007 * = Number of attributes for each block on the
1008 * current processor
1009 * globals.Proc_Num_Elem_In_Blk [globals.Num_Elem_Blk]
1010 * = Number of elements in the processor's
1011 * element blocks
1012 *
1013 * ------------------------------------------------------------------------
1014 *
1015 * Functions called:
1016 *
1017 * find_elem_block -- function which finds the element block which
1018 * owns each element in this processor. In addition,
1019 * it determines a map from the current element to an
1020 * element block.
1021 *
1022 * ------------------------------------------------------------------------
1023 */
1024
1025 {
1026 std::vector<INT> proc_elem_blk{};
1027
1028 /* Element blocks local to the current processor */
1029
1030 /**************************** execution begins ******************************/
1031
1032 /*
1033 * Allocate temporary array for a mapping between the local element number
1034 * and the corresponding element block id for the element block that the
1035 * element belongs to.
1036 */
1037
1038 for (int iproc = Proc_Info[4]; iproc < Proc_Info[4] + Proc_Info[5]; iproc++) {
1039
1040 proc_elem_blk.resize(globals.Num_Internal_Elems[iproc] + globals.Num_Border_Elems[iproc]);
1041
1042 /* Find out which element block each element in this processor belongs to.
1043 * Fill this information into the temporary vector, proc_elem_blk. Also,
1044 * calculate:
1045 *
1046 * globals.Num_Elem_Blk = Number of element blocks defined on
1047 * the current processor.
1048 * globals.GElem_Blks = Map from the local element block number
1049 * to the global element block number.
1050 */
1051 find_elem_block(proc_elem_blk, iproc, Proc_Ids[iproc]);
1052
1053 /* Allocate Permanent integer arrays, which have lengths equal to the
1054 * number of element blocks defined on the processor. This is done with a
1055 * single malloc call. Initialize this space to zero.
1056 */
1057 if (globals.Num_Elem_Blk > 0) {
1058 globals.Proc_Nodes_Per_Elem[iproc] = (INT *)array_alloc(
1059 __FILE__, __LINE__, 1, (4 * globals.Num_Elem_Blk + globals.Proc_Num_Elem_Blk[iproc]),
1060 sizeof(INT));
1061 globals.Proc_Elem_Blk_Ids[iproc] = globals.Proc_Nodes_Per_Elem[iproc] + globals.Num_Elem_Blk;
1062 globals.Proc_Elem_Blk_Types[iproc] = globals.Proc_Elem_Blk_Ids[iproc] + globals.Num_Elem_Blk;
1063 globals.Proc_Num_Attr[iproc] =
1064 globals.Proc_Elem_Blk_Types[iproc] + globals.Proc_Num_Elem_Blk[iproc];
1065 globals.Proc_Num_Elem_In_Blk[iproc] = globals.Proc_Num_Attr[iproc] + globals.Num_Elem_Blk;
1066 /* Initialize */
1067 for (int i = 0; i < (4 * globals.Num_Elem_Blk + globals.Proc_Num_Elem_Blk[iproc]); i++) {
1068 globals.Proc_Nodes_Per_Elem[iproc][i] = 0;
1069 }
1070 }
1071 else {
1072 fmt::print(stderr, "ERROR globals.Num_Elem_Blk = {}\n", globals.Num_Elem_Blk);
1073 exit(1);
1074 }
1075
1076 /*
1077 * Fill in the local processor element block arrays from a lookup from the
1078 * global element block arrays. Note that the global element block arrays
1079 * will be freed at the end of the file.
1080 */
1081 for (int i = 0; i < globals.Proc_Num_Elem_Blk[iproc]; i++) {
1082
1083 size_t iglobal_blk = globals.GElem_Blks[iproc][i];
1084 globals.Proc_Nodes_Per_Elem[iproc][i] = Num_Nodes_Per_Elem[iglobal_blk];
1085 globals.Proc_Elem_Blk_Ids[iproc][i] = Elem_Blk_Ids[iglobal_blk];
1086 globals.Proc_Num_Attr[iproc][i] = Num_Attr_Per_Elem[iglobal_blk];
1087
1088 /* Determine the element type integer id for this current element block.
1089 * The element type integer ID is a unique identifier (unique for all
1090 * element types that the program knows about). It is determined from
1091 * the character string name, Elem_Blk_Types, and the number of nodes in
1092 * the element.
1093 */
1094 globals.Proc_Elem_Blk_Types[iproc][i] = get_type(
1095 Elem_Blk_Types[iglobal_blk], globals.Proc_Nodes_Per_Elem[iproc][i], globals.Num_Dim);
1096 }
1097
1098 /*
1099 * Determine the number of elements in each of the element blocks defined
1100 * on the local processor.
1101 */
1102
1103 for (int i = 0; i < globals.Proc_Num_Elem_Blk[iproc]; i++) {
1104 for (INT j = 0; j < globals.Num_Internal_Elems[iproc] + globals.Num_Border_Elems[iproc];
1105 j++) {
1106 if (proc_elem_blk[j] == globals.Proc_Elem_Blk_Ids[iproc][i]) {
1107 (globals.Proc_Num_Elem_In_Blk[iproc][i])++;
1108 }
1109 }
1110 }
1111
1112 /* Sort globals.GElems so that each element block is monotonic */
1113 size_t j = 0;
1114 for (int i = 0; i < globals.Proc_Num_Elem_Blk[iproc]; i++) {
1115 gds_qsort((Data(globals.GElems[iproc])) + j, globals.Proc_Num_Elem_In_Blk[iproc][i]);
1116 j += globals.Proc_Num_Elem_In_Blk[iproc][i];
1117 }
1118 } /* End "for(iproc=0; iproc <Proc_Info[2]; iproc++)" */
1119
1120 if (Debug_Flag >= 5) {
1121 for (int iproc = Proc_Info[4]; iproc < Proc_Info[4] + Proc_Info[5]; iproc++) {
1122
1123 /* Printout the Element Block Information defined on each Processor */
1124 print_line("=", 79);
1125 fmt::print("\t\tLocal Element Block information for Proc = {}\n", Proc_Ids[iproc]);
1126 fmt::print("\t\tNumber of Elem blocks on processor = {}\n", globals.Proc_Num_Elem_Blk[iproc]);
1127 fmt::print("{}{}\n", "Local_Block_Num Global_Block_Num Block_ID Nodes_Per_Elem ",
1128 "Num_Attributes Elem_Blk_Type globals.Proc_Num_Elem_In_Blk "
1129 "Glb_Elm_In_Blk");
1130 print_line("-", 79);
1131 for (int i = 0; i < globals.Proc_Num_Elem_Blk[iproc]; i++) {
1132 fmt::print("{:4d}\t\t{:5}\t{:8}\t{:8}\t{:8}\t{:8}\t{:8}\t{:8}\n", i,
1133 globals.GElem_Blks[iproc][i],
1134 fmt::group_digits(globals.Proc_Elem_Blk_Ids[iproc][i]),
1135 fmt::group_digits(globals.Proc_Nodes_Per_Elem[iproc][i]),
1136 fmt::group_digits(globals.Proc_Num_Attr[iproc][i]),
1137 globals.Proc_Elem_Blk_Types[iproc][i],
1138 fmt::group_digits(globals.Proc_Num_Elem_In_Blk[iproc][i]),
1139 fmt::group_digits(Num_Elem_In_Blk[globals.GElem_Blks[iproc][i]]));
1140 }
1141 print_line("=", 79);
1142 }
1143 }
1144
1145 } /* END extract_elem_blk() */
1146
1147 /*****************************************************************************/
1148 /*****************************************************************************/
1149 /*****************************************************************************/
1150
1151 template <typename T, typename INT> void NemSpread<T, INT>::read_elem_blk(int exoid)
1152
1153 /* Function which reads the element block information from an EXODUS
1154 * database. If an error is tripped, the program will set a flag, and abort at
1155 * the end of the routine.
1156 *
1157 * ------------------------------------------------------------------------
1158 *
1159 * extract_elem_connect
1160 * - Function which sorts through the element block
1161 * connectivity information and extracts the
1162 * information required by each processor.
1163 *
1164 * ------------------------------------------------------------------------
1165 */
1166
1167 {
1168
1169 /* Local variables */
1170
1171 T *elem_attr = nullptr;
1172 #ifdef DEBUG
1173 size_t ielem_count;
1174 #endif
1175 INT *elem_blk = nullptr, ipos;
1176 size_t num_elem_per_message = 0;
1177 size_t num_attr_per_message = 0;
1178 size_t num_attr_left_over = 0;
1179 size_t num_elem_messages = 0;
1180 size_t num_attr_messages = 0;
1181 size_t num_elem_left_over = 0;
1182
1183 size_t istart_elem;
1184 size_t iend_elem;
1185 size_t istart_attr;
1186 size_t iend_attr;
1187 int local_ielem_blk;
1188
1189 /**************************** execution begins ******************************/
1190
1191 /*
1192 * Allocate memory for the vector of global element types in the entire
1193 * mesh. Used in sideset manipulations
1194 */
1195 GM_Elem_Types = (int *)array_alloc(__FILE__, __LINE__, 1, globals.Num_Elem, sizeof(int));
1196
1197 std::vector<INT> global_ids(globals.Num_Elem);
1198
1199 for (int iproc = Proc_Info[4]; iproc < Proc_Info[4] + Proc_Info[5]; iproc++) {
1200
1201 /*
1202 * Build globals.Proc_Connect_Ptr, a vector of pointers to the start of each
1203 * element's connectivity list in globals.Proc_Elem_Connect. The last entry in
1204 * globals.Proc_Connect_Ptr is the length of globals.Proc_Elem_Connect.
1205 */
1206 globals.Proc_Connect_Ptr[iproc] = (INT *)array_alloc(
1207 __FILE__, __LINE__, 1,
1208 globals.Num_Internal_Elems[iproc] + globals.Num_Border_Elems[iproc] + 1, sizeof(INT));
1209
1210 size_t iptr_count = 0;
1211 size_t iconnect_length = 0;
1212 for (int i = 0; i < globals.Proc_Num_Elem_Blk[iproc]; i++) {
1213 for (INT j = 0; j < globals.Proc_Num_Elem_In_Blk[iproc][i]; j++) {
1214 globals.Proc_Connect_Ptr[iproc][iptr_count++] = iconnect_length;
1215 iconnect_length += globals.Proc_Nodes_Per_Elem[iproc][i];
1216 }
1217 }
1218 globals.Proc_Connect_Ptr[iproc][iptr_count] = iconnect_length;
1219
1220 /*
1221 * Allocate the processor's connectivity list vector
1222 * - This is a global vector
1223 */
1224 globals.Proc_Elem_Connect[iproc] =
1225 (INT *)array_alloc(__FILE__, __LINE__, 1, iconnect_length, sizeof(INT));
1226
1227 #ifdef DEBUG
1228 for (size_t i = 0; i < iconnect_length; i++)
1229 globals.Proc_Elem_Connect[iproc][i] = -1111111;
1230 #endif
1231
1232 /*
1233 * Allocate the processor's attribute list vector, if its length
1234 * is nonzero
1235 */
1236 size_t iattr_length = 0;
1237 for (int i = 0; i < globals.Proc_Num_Elem_Blk[iproc]; i++) {
1238 iattr_length += globals.Proc_Num_Attr[iproc][i] * globals.Proc_Num_Elem_In_Blk[iproc][i];
1239 }
1240 if (iattr_length > 0) {
1241 globals.Proc_Elem_Attr[iproc].resize(iattr_length);
1242 }
1243 } /* End "for(iproc=0; iproc <Proc_Info[2]; iproc)" */
1244
1245 size_t icount = 0;
1246
1247 /*
1248 * READ THE ELEMENT CONNECTIVITY AND ATTRIBUTE INFORMATION
1249 * FROM THE EXODUS FILE FOR EACH ELEMENT BLOCK
1250 */
1251 for (int ielem_blk = 0; ielem_blk < globals.Num_Elem_Blk; ielem_blk++) {
1252 if (Num_Elem_In_Blk[ielem_blk] > 0) {
1253
1254 /*
1255 * Calculate the size of a single element's connectivity list and a
1256 * single element's attribute list
1257 */
1258 size_t iconnect_size = Num_Nodes_Per_Elem[ielem_blk] * sizeof(INT);
1259 size_t iattr_size = Num_Attr_Per_Elem[ielem_blk] * sizeof(T);
1260
1261 /*
1262 * Determine how many element connectivity list and how many element
1263 * attributes can be send in single message
1264 */
1265 find_message_info(iconnect_size, Num_Elem_In_Blk[ielem_blk], &num_elem_per_message,
1266 &num_elem_messages, &num_elem_left_over);
1267
1268 if (iattr_size > 0) {
1269 find_message_info(iattr_size, Num_Elem_In_Blk[ielem_blk], &num_attr_per_message,
1270 &num_attr_messages, &num_attr_left_over);
1271 }
1272 else {
1273 num_attr_messages = 0;
1274 }
1275
1276 if (Debug_Flag > 1) {
1277 fmt::print("\n\nMessage summary for Element Block number {}, ", ielem_blk);
1278 fmt::print("having a block id of {}:\n", Elem_Blk_Ids[ielem_blk]);
1279 fmt::print("\tNumber of messages needed for the element connectivity "
1280 "vector = {}\n",
1281 num_elem_messages);
1282 fmt::print("\tNumber of elements per message = {}\n", num_elem_per_message);
1283 fmt::print("\tNumber of nodes per element = {}\n", Num_Nodes_Per_Elem[ielem_blk]);
1284 fmt::print("\tLength of each message = {} bytes\n",
1285 (Num_Nodes_Per_Elem[ielem_blk] * num_elem_per_message * sizeof(INT)));
1286 if (num_attr_messages > 0) {
1287 fmt::print("\tNumber of attribute messages: {}\n\tNumber "
1288 "of attributes per message: {}\n\n",
1289 num_attr_messages, num_attr_per_message);
1290 }
1291 }
1292
1293 /*
1294 * Allocate the arrays for reading from the ExodusII file and
1295 * broadcasting
1296 */
1297 elem_blk = (INT *)array_alloc(
1298 __FILE__, __LINE__, 1, Num_Nodes_Per_Elem[ielem_blk] * num_elem_per_message, sizeof(INT));
1299 if (num_attr_messages > 0) {
1300 elem_attr = (T *)array_alloc(
1301 __FILE__, __LINE__, 1, Num_Attr_Per_Elem[ielem_blk] * num_attr_per_message, sizeof(T));
1302 }
1303 else {
1304 elem_attr = nullptr;
1305 }
1306
1307 /*
1308 * Read in the element connectivity list for the current element block,
1309 * ielem_blk, from Proc 0, and then, broadcast it to all of the processors
1310 */
1311 for (size_t i = 0; i < num_elem_messages; i++) {
1312
1313 if (Debug_Flag >= 2) {
1314 fmt::print("\telem block message: {} of {}\n", i + 1, num_elem_messages);
1315 }
1316
1317 /* Initialize the element connectivity list to a value of -1.0 */
1318 for (size_t j = 0; j < Num_Nodes_Per_Elem[ielem_blk] * num_elem_per_message;
1319 elem_blk[j++] = -1) {
1320 ;
1321 }
1322
1323 size_t num_to_get = 0;
1324 istart_elem = i * num_elem_per_message;
1325 if (num_elem_left_over == 0 || i < num_elem_messages - 1) {
1326 num_to_get = num_elem_per_message;
1327 }
1328 else {
1329 num_to_get = num_elem_left_over;
1330 }
1331 iend_elem = istart_elem + num_to_get;
1332
1333 int el_type;
1334 check_exodus_error(ex_get_partial_conn(exoid, EX_ELEM_BLOCK, Elem_Blk_Ids[ielem_blk],
1335 (istart_elem + 1), num_to_get, elem_blk, nullptr,
1336 nullptr),
1337 "ex_get_partial_conn");
1338 if (Debug_Flag >= 2) {
1339 fmt::print("\t\tread connectivity\n");
1340 }
1341
1342 el_type =
1343 get_type(Elem_Blk_Types[ielem_blk], Num_Nodes_Per_Elem[ielem_blk], globals.Num_Dim);
1344 for (size_t ielem = 0; ielem < num_to_get; ielem++) {
1345 GM_Elem_Types[icount++] = el_type;
1346 }
1347
1348 if (Debug_Flag >= 2) {
1349 fmt::print("\t\tgot element types\n");
1350 }
1351
1352 /* PRINT OUT THE ELEMENT CONNECTIVITY TABLE IF IN DEBUGGING MODE */
1353 if (Debug_Flag >= 6) {
1354 fmt::print("\n\n\n");
1355 print_line("=", 79);
1356 fmt::print("Printout of Element connectivity list obtained from "
1357 "Exodus file:\n");
1358 fmt::print("\tGlobal element block number = {}\n", ielem_blk);
1359 fmt::print("\tElement ID number = {}\n", Elem_Blk_Ids[ielem_blk]);
1360 fmt::print("\tMessage number = {}\n", i);
1361 print_line("-", 79);
1362 ipos = 0;
1363 for (size_t j = 0; j < num_to_get; j++) {
1364 fmt::print("\t elem: {}, nodes:", j);
1365 for (int k = 0; k < Num_Nodes_Per_Elem[ielem_blk]; k++) {
1366 fmt::print(" {}", elem_blk[ipos++]);
1367 }
1368 fmt::print("\n");
1369 }
1370 print_line("=", 79);
1371 }
1372
1373 /*
1374 * On each processor, extract the element connectivity lists that the
1375 * processor needs
1376 */
1377 for (int iproc = Proc_Info[4]; iproc < Proc_Info[4] + Proc_Info[5]; iproc++) {
1378 extract_elem_connect(elem_blk, ielem_blk, istart_elem, iend_elem, &local_ielem_blk,
1379 iproc);
1380 }
1381 if (Debug_Flag >= 2) {
1382 fmt::print("\t\textract connectivity\n");
1383 }
1384
1385 } /* End "for(i=0; i < num_elem_messages; i++)" */
1386
1387 /* Read in the element attribute lists and broadcast to the processors */
1388 for (size_t i = 0; i < num_attr_messages; i++) {
1389
1390 if (Debug_Flag >= 2) {
1391 fmt::print("\tattribute message: {} of {}\n", i + 1, num_attr_messages);
1392 }
1393
1394 /* Initialize */
1395 for (size_t j = 0; j < Num_Attr_Per_Elem[ielem_blk] * num_attr_per_message; j++) {
1396 elem_attr[j] = 0.0;
1397 }
1398
1399 istart_attr = i * num_attr_per_message;
1400
1401 if (num_attr_left_over == 0 || i < (num_attr_messages - 1)) {
1402 iend_attr = istart_attr + num_attr_per_message;
1403 }
1404 else {
1405 iend_attr = istart_attr + num_attr_left_over;
1406 }
1407
1408 if (num_attr_left_over == 0 || i < (num_attr_messages - 1)) {
1409 check_exodus_error(ex_get_partial_attr(exoid, EX_ELEM_BLOCK, Elem_Blk_Ids[ielem_blk],
1410 (istart_attr + 1), num_attr_per_message,
1411 elem_attr),
1412 "ex_get_partial_attr");
1413 }
1414 else {
1415 check_exodus_error(ex_get_partial_attr(exoid, EX_ELEM_BLOCK, Elem_Blk_Ids[ielem_blk],
1416 (istart_attr + 1), num_attr_left_over, elem_attr),
1417 "ex_get_partial_attr");
1418 }
1419
1420 for (int iproc = Proc_Info[4]; iproc < Proc_Info[4] + Proc_Info[5]; iproc++) {
1421
1422 if (Debug_Flag > 6) {
1423 fmt::print("\t\tExtract attributes for processor {}\n", Proc_Ids[iproc]);
1424 }
1425 /*
1426 * On each processor, extract the element attributes that the
1427 * processor needs
1428 */
1429 extract_elem_attr(elem_attr, ielem_blk, istart_attr, iend_attr,
1430 Num_Attr_Per_Elem[ielem_blk], iproc);
1431 }
1432 }
1433
1434 /* Free vectors */
1435 safe_free((void **)&elem_blk);
1436 safe_free((void **)&elem_attr);
1437
1438 } /* END "if (globals.Proc_Num_Elem_In_Blk[ielem_blk] > 0)" */
1439
1440 } /* End "for(ielem_blk=0; ielem_blk < globals.Num_Elem_Blk; ielem_blk++)" */
1441
1442 /* Handle global element ids... */
1443 check_exodus_error(ex_get_id_map(exoid, EX_ELEM_MAP, Data(global_ids)), "ex_get_id_map");
1444
1445 /*
1446 * Check whether map is sequential (1..globals.Num_Elem). If it is, then it
1447 * provides no information and we don't need to store it in the
1448 * output databases.
1449 */
1450 {
1451 bool sequential = true;
1452 for (size_t i = 0; i < globals.Num_Elem; i++) {
1453 if ((size_t)global_ids[i] != i + 1) {
1454 sequential = false;
1455 break;
1456 }
1457 }
1458
1459 // Check that map is valid 1 <= global_node_id[*]
1460 // If not, output a warning and disable the map.
1461 for (size_t i = 0; i < globals.Num_Elem; i++) {
1462 if (global_ids[i] <= 0) {
1463 fmt::print(stderr,
1464 "---------------------------------------------------------------------\n"
1465 "ERROR: Local element {} has a global id of {} which is invalid.\n"
1466 " All global ids must be greater than 0. The map will be ignored.\n"
1467 "---------------------------------------------------------------------\n",
1468 fmt::group_digits(i + 1), fmt::group_digits(global_ids[i]));
1469 sequential = true; // Map is invalid, ignore it.
1470 break;
1471 }
1472 }
1473
1474 if (!sequential) {
1475 for (int iproc = Proc_Info[4]; iproc < Proc_Info[4] + Proc_Info[5]; iproc++) {
1476 globals.Proc_Global_Elem_Id_Map[iproc].resize(globals.Num_Internal_Elems[iproc] +
1477 globals.Num_Border_Elems[iproc]);
1478 extract_global_element_ids(global_ids, globals.Num_Elem, iproc);
1479 }
1480 }
1481 else {
1482 /* Should be nullptr already, but make it more clear */
1483 for (int iproc = Proc_Info[4]; iproc < Proc_Info[4] + Proc_Info[5]; iproc++) {
1484 globals.Proc_Global_Elem_Id_Map[iproc].clear();
1485 }
1486 }
1487 }
1488 } /* read_elem_blk */
1489
1490 /*****************************************************************************/
1491 /*****************************************************************************/
1492 /*****************************************************************************/
1493
1494 template <typename T, typename INT>
1495 void NemSpread<T, INT>::extract_global_element_ids(const std::vector<INT> &global_ids,
1496 size_t /*Num_Elem*/, int iproc)
1497 {
1498 /*
1499 * globals.Num_Elem -- number of elements in serial mesh.
1500 * global_ids[] (size globals.Num_Elem) = global id corresponding to index
1501 * of element in serial file
1502 * globals.Num_Internal_Elems[iproc] + globals.Num_Border_Elems[iproc] = number of element on this
1503 processor
1504 * globals.GElems[iproc][i] = index of element in serial file corresponding
1505 to i'th element on this processor. Both are zero-based?
1506 */
1507 size_t num_local_element = globals.Num_Internal_Elems[iproc] + globals.Num_Border_Elems[iproc];
1508 for (size_t i = 0; i < num_local_element; i++) {
1509 globals.Proc_Global_Elem_Id_Map[iproc][i] = global_ids[globals.GElems[iproc][i]];
1510 }
1511 }
1512
1513 /*****************************************************************************/
1514 /*****************************************************************************/
1515 /*****************************************************************************/
1516
1517 template <typename T, typename INT>
1518 void NemSpread<T, INT>::extract_global_node_ids(const std::vector<INT> &global_ids,
1519 size_t /*Num_Node*/, int iproc)
1520 {
1521 /*
1522 * Num_Elem -- number of elements in serial mesh.
1523 * global_ids[] (size Num_Elem) = global id corresponding to index
1524 * of element in serial file
1525 * Num_Internal_Elems[iproc] + Num_Border_Elems[iproc] = number of element on this processor
1526 * GElems[iproc][i] = index of element in serial file corresponding
1527 * to i'th element on this processor. Both are zero-based?
1528 */
1529 size_t num_local_node = globals.Num_Internal_Nodes[iproc] + globals.Num_Border_Nodes[iproc] +
1530 globals.Num_External_Nodes[iproc];
1531 for (size_t i = 0; i < num_local_node; i++) {
1532 globals.Proc_Global_Node_Id_Map[iproc][i] = global_ids[globals.GNodes[iproc][i]];
1533 }
1534 }
1535
1536 /*****************************************************************************/
1537 /*****************************************************************************/
1538 /*****************************************************************************/
1539
1540 template <typename T, typename INT>
1541 size_t NemSpread<T, INT>::extract_elem_connect(INT elem_blk[], int icurrent_elem_blk,
1542 size_t istart_elem, size_t iend_elem,
1543 int *local_ielem_blk, int iproc)
1544
1545 /*
1546 * Function which extracts the element connectivity list from the current
1547 * element connectivity message. The results are put into the global
1548 * vector, globals.Proc_Elem_Connect.
1549 *
1550 * The function returns the number of elements in the current message
1551 * that are defined on the current processor.
1552 */
1553
1554 {
1555 size_t count_hits = 0;
1556 bool found = false;
1557
1558 /* Match the Element Block Id of the global block with the local block number
1559 * The end result is the current value of ielem_blk, Store this value in the
1560 * output variable, local_ielem_blk.
1561 */
1562 for (int ielem_blk = 0; ielem_blk < globals.Proc_Num_Elem_Blk[iproc]; ielem_blk++) {
1563 if (globals.Proc_Elem_Blk_Ids[iproc][ielem_blk] == Elem_Blk_Ids[icurrent_elem_blk]) {
1564 *local_ielem_blk = ielem_blk;
1565 found = true;
1566
1567 /* Some elements in the current element block passed in from
1568 read_elem_blk may be on this processor */
1569
1570 /* Calculate the number of elements in the current message */
1571 size_t num_elem = iend_elem - istart_elem;
1572
1573 /* Calculate iglobal_offset - The sum of the elements in all the global
1574 * element blocks preceding this one
1575 */
1576 size_t iglobal_offset = 0;
1577 for (int i = 0; i < globals.GElem_Blks[iproc][ielem_blk]; i++) {
1578 iglobal_offset += Num_Elem_In_Blk[i];
1579 }
1580
1581 /* Calculate iproc_offset - The sum of the elements in all the
1582 * processor's
1583 * element blocks preceding this one.
1584 * Calculate iproc_start - The starting position in the connectivity
1585 * vector for this element block,
1586 * globals.Proc_Elem_Connect[]
1587 */
1588 size_t iproc_offset = 0;
1589 size_t iproc_start = 0;
1590 for (int i = 0; i < ielem_blk; i++) {
1591 iproc_offset += globals.Proc_Num_Elem_In_Blk[iproc][i];
1592 iproc_start +=
1593 globals.Proc_Num_Elem_In_Blk[iproc][i] * globals.Proc_Nodes_Per_Elem[iproc][i];
1594 }
1595
1596 INT iglobal_begin = iglobal_offset + istart_elem;
1597 INT iglobal_end = iglobal_begin + num_elem;
1598
1599 size_t ibegin = iproc_offset;
1600 size_t iend = iproc_offset + globals.Proc_Num_Elem_In_Blk[iproc][ielem_blk];
1601
1602 /* OPTIMIZE: Check that max globals.GElems is >min < max...*/
1603 /* Note that globals.GElems is sorted, so we can check that the
1604 * ranges overlap and skip the loop if they don't */
1605 if (globals.GElems[iproc][ibegin] < iglobal_end &&
1606 globals.GElems[iproc][iend - 1] >= iglobal_begin) {
1607
1608 for (size_t i = ibegin; i < iend; i++) {
1609 INT iglobal_elem = globals.GElems[iproc][i];
1610
1611 /* globals.GElems is sorted, so if we are outside the iglobal range,
1612 * break out of this loop...
1613 */
1614 if (iglobal_elem > iglobal_end) {
1615 break;
1616 }
1617
1618 if (iglobal_begin <= iglobal_elem && iglobal_elem < iglobal_end) {
1619 count_hits++;
1620
1621 size_t ipos =
1622 iproc_start + (i - iproc_offset) * globals.Proc_Nodes_Per_Elem[iproc][ielem_blk];
1623
1624 size_t iglobal_pos =
1625 (iglobal_elem - iglobal_begin) * globals.Proc_Nodes_Per_Elem[iproc][ielem_blk];
1626
1627 assert(iglobal_pos < num_elem * globals.Proc_Nodes_Per_Elem[iproc][ielem_blk]);
1628 /*
1629 * Store the connectivity information for the current element into
1630 * globals.Proc_Elem_Connect []. At the same time, Decrement by one the
1631 * value of all nodes in the connectivity matrix.
1632 */
1633 for (int j = 0; j < globals.Proc_Nodes_Per_Elem[iproc][ielem_blk]; j++) {
1634 globals.Proc_Elem_Connect[iproc][ipos++] = elem_blk[iglobal_pos++] - 1;
1635 }
1636 }
1637 }
1638 }
1639 } /* END if (globals.Proc_Elem_Blk_Ids[ielem_blk]== ... */
1640 }
1641
1642 if (!found) {
1643 *local_ielem_blk = -1;
1644 }
1645
1646 return count_hits;
1647
1648 } /* extract_elem_connect */
1649
1650 /*****************************************************************************/
1651 /*****************************************************************************/
1652 /*****************************************************************************/
1653
1654 template <typename T, typename INT>
1655 void NemSpread<T, INT>::extract_elem_attr(T *elem_attr, int icurrent_elem_blk, size_t istart_elem,
1656 size_t iend_elem, int natt_p_elem, int iproc)
1657 {
1658 /*
1659 * Function which extracts the element connectivity list from the current
1660 * element connectivity message. The results are put into the global
1661 * vector, globals.Proc_Elem_Connect. This function is run on all processors,
1662 * include Proc = 0
1663 *
1664 * Author: Scott Hutchinson (1421)
1665 *
1666 * ------------------------------------------------------------------
1667 *
1668 * input
1669 * -----------
1670 * elem_attr = Vector containing the element attributes for this message.
1671 * This message consists of the either the whole or
1672 * part of the element attribute array for a single
1673 * element block.
1674 *
1675 * istart_elem = This is the starting value of the element number for
1676 * the element attribute message. i.e., the first
1677 * message in an element block will have
1678 * istart_elem = 0. The second message for that
1679 * element block will have the value:
1680 * istart_elem = num_elem_per_message.
1681 * iend_elem = This is the value + 1 of the last element number for
1682 * the element attribute message. i.e., the first
1683 * message in an element block will have
1684 * iend_elem = num_elem_per_message
1685 */
1686
1687 /* Match the Element Block Id of the global block with the local block number */
1688 for (int ielem_blk = 0; ielem_blk < globals.Proc_Num_Elem_Blk[iproc]; ielem_blk++) {
1689 if (globals.Proc_Elem_Blk_Ids[iproc][ielem_blk] == Elem_Blk_Ids[icurrent_elem_blk]) {
1690
1691 /* Some elements in the current element block passed in from
1692 read_elem_blk may be on this processor */
1693
1694 /* Calculate the number of elements in the current message */
1695 size_t num_elem = iend_elem - istart_elem;
1696
1697 /* Calculate iglobal_offset - The sum of the elements in all the global
1698 * element blocks preceding this one
1699 */
1700 size_t iglobal_offset = 0;
1701 for (int i = 0; i < globals.GElem_Blks[iproc][ielem_blk]; i++) {
1702 iglobal_offset += Num_Elem_In_Blk[i];
1703 }
1704
1705 /* Calculate iproc_offset - The sum of the elements in all the
1706 * processor's
1707 * element blocks preceding this one.
1708 * Calculate iproc_start - The starting position in the connectivity
1709 * vector for this element block,
1710 * globals.Proc_Elem_Connect[]
1711 */
1712 size_t iproc_offset = 0;
1713 size_t iproc_start = 0;
1714 for (int i = 0; i < ielem_blk; i++) {
1715 iproc_offset += globals.Proc_Num_Elem_In_Blk[iproc][i];
1716 iproc_start += globals.Proc_Num_Elem_In_Blk[iproc][i] * globals.Proc_Num_Attr[iproc][i];
1717 }
1718
1719 INT iglobal_begin = iglobal_offset + istart_elem;
1720 INT iglobal_end = iglobal_begin + num_elem;
1721
1722 size_t ibegin = iproc_offset;
1723 size_t iend = iproc_offset + globals.Proc_Num_Elem_In_Blk[iproc][ielem_blk];
1724
1725 /* OPTIMIZE: Check that max globals.GElems is >min < max...*/
1726 /* Note that globals.GElems is sorted, so we can check that the
1727 * ranges overlap and skip the loop if they don't */
1728 if (globals.GElems[iproc][ibegin] < iglobal_end &&
1729 globals.GElems[iproc][iend - 1] >= iglobal_begin) {
1730
1731 for (size_t i = ibegin; i < iend; i++) {
1732 INT iglobal_elem = globals.GElems[iproc][i];
1733
1734 /* globals.GElems is sorted, so if we are outside the iglobal range,
1735 * break out of this loop...
1736 */
1737 if (iglobal_elem > iglobal_end) {
1738 break;
1739 }
1740
1741 if (iglobal_begin <= iglobal_elem && iglobal_elem < iglobal_end) {
1742 size_t ipos = iproc_start + (i - iproc_offset) * natt_p_elem;
1743 size_t iglobal_pos = (iglobal_elem - iglobal_begin) * natt_p_elem;
1744
1745 assert(iglobal_pos < num_elem * natt_p_elem);
1746 /*
1747 * Store the connectivity information for the current element into
1748 * globals.Proc_Elem_Connect []. At the same time, Decrement by one the
1749 * value of all nodes in the connectivity matrix.
1750 */
1751 for (int j = 0; j < natt_p_elem; j++) {
1752 globals.Proc_Elem_Attr[iproc][ipos++] = elem_attr[iglobal_pos++];
1753 }
1754 }
1755 }
1756 }
1757 } /* END if (globals.Proc_Elem_Blk_Ids[ielem_blk]== ... */
1758 }
1759 } /* extract_elem_connect */
1760
1761 /*****************************************************************************/
1762 /*****************************************************************************/
1763 /*****************************************************************************/
1764
1765 template <typename T, typename INT>
1766 void NemSpread<T, INT>::find_elem_block(std::vector<INT> &proc_elem_blk, int iproc,
1767 int /*proc_for*/)
1768
1769 {
1770 /* Function which finds the element block which owns each element on the
1771 * current processor. In addition, a map from the local element block number
1772 * to the global element block number, *globals.GElem_Blks, is created.
1773 * globals.Num_Elem_Blk, the number of element blocks on the current processor,
1774 * is calculated This function is called by every processor.
1775 *
1776 * Author(s): Scott Hutchinson (9221)
1777 * Gary Hennigan (9221)
1778 *
1779 * Output:
1780 *
1781 * globals.Num_Elem_Blk = Number of element blocks on the processor
1782 * (Global int variable)
1783 *
1784 * *proc_elem_blk = Vector of element block ids for the local
1785 * elements defined on the processor.
1786 * (Local int vector of length
1787 * globals.Num_Internal_Elems + globals.Num_Border_Elems)
1788 *
1789 * *globals.GElem_Blks = Map from the local element block number to
1790 * the global element block number.
1791 * (Global int vector of length globals.Num_Elem_Blk)
1792 */
1793 if (globals.Num_Elem_Blk == 0) {
1794 return;
1795 }
1796
1797 /* Boolean vector of length globals.Num_Elem_Blk If the i'th
1798 element block exists on the current processor, the i'th entry
1799 is set to TRUE */
1800 std::vector<bool> elem_in_blk(globals.Num_Elem_Blk);
1801
1802 /* Vector of integer offsets into the vector globals.GElems.
1803 It has a length of globals.Num_Elem_Blk+1. The i'th entry
1804 points to the beginning of of the element map information
1805 for the first element in the i'th element block. The
1806 (i+1)th entry points to the last element for the i'th block
1807 in the vector globals.GElem. */
1808 std::vector<INT> elem_blk_point(globals.Num_Elem_Blk + 1);
1809
1810 /*
1811 * Construct a vector of pointers to the beginning of element blocks in an
1812 * element connectivity list (vector)
1813 */
1814 elem_blk_point[0] = 0;
1815 for (int i = 0; i < globals.Num_Elem_Blk; i++) {
1816 elem_blk_point[i + 1] = elem_blk_point[i] + Num_Elem_In_Blk[i];
1817 }
1818
1819 /*
1820 * Find out which elements belonging to the current processor are in which
1821 * local element block. Store this in *proc_elem_blk.
1822 */
1823
1824 /* Internal Elements */
1825 if (check_monot(&globals.GElems[iproc][0], globals.Num_Internal_Elems[iproc])) {
1826 size_t tmp_cnt = globals.Num_Internal_Elems[iproc];
1827 int j = 0;
1828 size_t i = 0;
1829 while (i < tmp_cnt && j < globals.Num_Elem_Blk) {
1830 while (i < tmp_cnt && globals.GElems[iproc][i] < elem_blk_point[j + 1]) {
1831 assert(globals.GElems[iproc][i] >= elem_blk_point[j]);
1832 proc_elem_blk[i++] = j;
1833 elem_in_blk[j] = true;
1834 }
1835 j++;
1836 }
1837 }
1838 else {
1839 for (INT i = 0; i < globals.Num_Internal_Elems[iproc]; i++) {
1840 bool found = false;
1841 for (int j = 0; j < globals.Num_Elem_Blk && !found; j++) {
1842 if (globals.GElems[iproc][i] < elem_blk_point[j + 1] &&
1843 globals.GElems[iproc][i] >= elem_blk_point[j]) {
1844 proc_elem_blk[i] = j;
1845 elem_in_blk[j] = true;
1846 found = true;
1847 }
1848 }
1849 if (!found) {
1850 fmt::print(stderr, "find_elem_block: Error!:\n");
1851 fmt::print(stderr,
1852 "\tElement {} not found in any element "
1853 "block.\n",
1854 i);
1855 exit(1);
1856 }
1857 }
1858 }
1859
1860 /* Border Elements */
1861 if (check_monot(&globals.GElems[iproc][globals.Num_Internal_Elems[iproc]],
1862 globals.Num_Border_Elems[iproc])) {
1863 INT tmp_cnt = globals.Num_Internal_Elems[iproc] + globals.Num_Border_Elems[iproc];
1864 int j = 0;
1865 INT i = globals.Num_Internal_Elems[iproc];
1866 while (i < tmp_cnt && j < globals.Num_Elem_Blk) {
1867 while (i < tmp_cnt && globals.GElems[iproc][i] < elem_blk_point[j + 1]) {
1868 assert(globals.GElems[iproc][i] >= elem_blk_point[j]);
1869 proc_elem_blk[i++] = j;
1870 elem_in_blk[j] = true;
1871 }
1872 j++;
1873 }
1874 }
1875 else {
1876 INT tmp_cnt = globals.Num_Internal_Elems[iproc] + globals.Num_Border_Elems[iproc];
1877 for (INT i = globals.Num_Internal_Elems[iproc]; i < tmp_cnt; i++) {
1878 bool found = false;
1879 for (int j = 0; j < globals.Num_Elem_Blk && !found; j++) {
1880 if (globals.GElems[iproc][i] < elem_blk_point[j + 1] &&
1881 globals.GElems[iproc][i] >= elem_blk_point[j]) {
1882 proc_elem_blk[i] = j;
1883 elem_in_blk[j] = true;
1884 found = true;
1885 }
1886 }
1887 if (!found) {
1888 fmt::print(stderr, "find_elem_block: Error!:\n");
1889 fmt::print(stderr,
1890 "\tElement {} not found in any element "
1891 "block.\n",
1892 i);
1893 exit(1);
1894 }
1895 }
1896 }
1897
1898 /*
1899 * Reorder globals.GElems based on the element block information. This is necessary
1900 * to insure that internal and border elements that are in the same element
1901 * block are sequentially contained in globals.GElems, which is assumed in some
1902 * later operations.
1903 */
1904
1905 my_sort(globals.Num_Internal_Elems[iproc] + globals.Num_Border_Elems[iproc], Data(proc_elem_blk),
1906 Data(globals.GElems[iproc]));
1907
1908 /* Now change proc_elem_blk to be a list of global element block IDs */
1909 for (INT i = 0; i < globals.Num_Internal_Elems[iproc] + globals.Num_Border_Elems[iproc]; i++) {
1910 size_t j = proc_elem_blk[i];
1911 proc_elem_blk[i] = Elem_Blk_Ids[j];
1912 }
1913
1914 /* Count the number of element blocks defined on this processor */
1915 globals.Proc_Num_Elem_Blk[iproc] = 0;
1916 for (int i = 0; i < globals.Num_Elem_Blk; i++) {
1917 if (elem_in_blk[i]) {
1918 globals.Proc_Num_Elem_Blk[iproc]++;
1919 }
1920 }
1921
1922 /*
1923 * Create a map from the current processor's element block number to the
1924 * 'global' element block number, globals.GElem_Blks. This is a permanent Global
1925 * Map.
1926 */
1927 globals.GElem_Blks[iproc] =
1928 (INT *)array_alloc(__FILE__, __LINE__, 1, globals.Num_Elem_Blk, sizeof(INT));
1929
1930 for (int i = 0, icount = 0; i < globals.Num_Elem_Blk; i++) {
1931 if (elem_in_blk[i]) {
1932 globals.GElem_Blks[iproc][icount++] = i;
1933 }
1934 }
1935 } /* END of routine find_elem_block() ****************************************/
1936 /*****************************************************************************/
1937 /*****************************************************************************/
1938 /*****************************************************************************/
1939
1940 template <typename T, typename INT>
1941 void NemSpread<T, INT>::read_node_sets(int exoid, std::vector<INT> &num_nodes_in_node_set,
1942 std::vector<INT> &num_df_in_nsets)
1943
1944 /* Function which reads the node sets information from an EXODUS database.
1945 * It read information in chunks. Then, it broadcasts the chunk to all
1946 * processors. Each processor, then searches for the chunk for what it is
1947 * responsible for. This function is called by all processors.
1948 *
1949 * Global Variables which are set by this routine
1950 * -----------------------------------------------
1951 *
1952 * globals.Proc_Num_Node_Sets = Number of node sets on the current processor
1953 *
1954 * globals.GNode_Sets [globals.Proc_Num_Node_Sets]
1955 * = Mapping between the local node set number and the
1956 * global node set number.
1957 *
1958 * globals.S_Ids [globals.Proc_Num_Node_Sets]
1959 * = Set IDs for the node sets defined on the current
1960 * processor
1961 *
1962 * globals.S_Count [globals.Proc_Num_Node_Sets]
1963 * = Number of nodes in the node sets defined on
1964 * the current processor
1965 *
1966 * globals.S_Pointers [globals.Proc_Num_Node_Sets]
1967 * = Pointers into the node set list for the node sets
1968 * defined on the current processor.
1969 *
1970 * globals.NS_List_Length = Length of the local node set list.
1971 *
1972 * globals.NS_List [globals.NS_List_Length]
1973 * = Concatenated node set list for the current
1974 * processor.
1975 */
1976
1977 {
1978 size_t num_messages;
1979 size_t num_left_over;
1980 size_t num_node_per_message;
1981
1982 /* Allocate arrays */
1983 std::vector<INT> list_length(Proc_Info[2]);
1984 std::vector<INT> proc_num_ns(Proc_Info[2]);
1985 std::vector<INT> ns_cntr(Proc_Info[2]);
1986 std::vector<bool> ns_on_proc(Proc_Info[2]);
1987
1988 /* pointers into the concatenated node set list which locate the start of the node sets */
1989 std::vector<std::vector<INT>> proc_list_pointer(Proc_Info[2]);
1990
1991 /* mapping from the processor node set numbers to the global node set numbers */
1992 std::vector<std::vector<INT>> ns_proc2glob(Proc_Info[2]);
1993
1994 /* the number of nodes in each processor's node set */
1995 std::vector<std::vector<INT>> proc_list_count(Proc_Info[2]);
1996
1997 /* the number of df's in each processor's node set (usually either zero or proc_list_count)*/
1998 std::vector<std::vector<INT>> proc_list_df_count(Proc_Info[2]);
1999
2000 /* the node number lists for the current node set for the current processor */
2001 std::vector<std::vector<std::vector<INT>>> proc_list(Proc_Info[2]);
2002
2003 /* Vector of pointers to nodeset global nodeeset node map lists.
2004 There is one pointer for each nodeset */
2005 std::vector<std::vector<std::vector<INT>>> proc_gmap_list(Proc_Info[2]);
2006
2007 /* the dist. factors for the current processor */
2008 std::vector<std::vector<std::vector<T>>> proc_list_df(Proc_Info[2]);
2009
2010 /* Initialize */
2011 for (int iproc = 0; iproc < Proc_Info[2]; iproc++) {
2012 ns_proc2glob[iproc].resize(globals.Num_Node_Set);
2013 proc_list_pointer[iproc].resize(globals.Num_Node_Set);
2014 proc_list_count[iproc].resize(globals.Num_Node_Set);
2015 proc_list_df_count[iproc].resize(globals.Num_Node_Set);
2016
2017 proc_list[iproc].resize(globals.Num_Node_Set);
2018 proc_gmap_list[iproc].resize(globals.Num_Node_Set);
2019 proc_list_df[iproc].resize(globals.Num_Node_Set);
2020 }
2021
2022 std::vector<INT> node_proc_index(globals.Num_Node + 1);
2023 for (int iproc = 0; iproc < Proc_Info[2]; iproc++) {
2024 size_t size = globals.Num_Internal_Nodes[iproc] + globals.Num_Border_Nodes[iproc];
2025 for (size_t i = 0; i < size; i++) {
2026 size_t ind = globals.GNodes[iproc][i];
2027 node_proc_index[ind]++;
2028 }
2029 }
2030
2031 /* Now convert the node_proc_index counts to an index... */
2032 size_t index = 0;
2033 for (size_t i = 0; i < globals.Num_Node; i++) {
2034 size_t count = node_proc_index[i];
2035 node_proc_index[i] = index;
2036 index += count;
2037 }
2038 node_proc_index[globals.Num_Node] = index;
2039
2040 /* Create a map from 'global node id' to processor */
2041 std::vector<INT> gmap(index, -1);
2042
2043 /* Fill it in....
2044 * Note that since nodes can be on multiple processors, we can't
2045 * normally do a simple one-to-one map of node to its processor.
2046 *
2047 * The values in gmap[] from node_proc_index[node] to node_proc_index[node+1]
2048 * contains the listing of the processors that node 'node' are on.
2049 */
2050 for (int iproc = 0; iproc < Proc_Info[2]; iproc++) {
2051 size_t size = globals.Num_Internal_Nodes[iproc] + globals.Num_Border_Nodes[iproc];
2052 for (size_t i = 0; i < size; i++) {
2053 size_t node = globals.GNodes[iproc][i];
2054 size_t beg = node_proc_index[node];
2055 size_t end = node_proc_index[node + 1];
2056 for (size_t j = beg; j < end; j++) {
2057 if (gmap[j] == -1) {
2058 gmap[j] = iproc;
2059 break;
2060 }
2061 }
2062 }
2063 }
2064
2065 /*-------------------------------------------------------------------------*/
2066 /* LOOP OVER THE NODE SETS */
2067 /*-------------------------------------------------------------------------*/
2068
2069 for (int i = 0; i < globals.Num_Node_Set; i++) {
2070 if (num_nodes_in_node_set[i] > 0) {
2071
2072 for (int iproc = 0; iproc < Proc_Info[2]; iproc++) {
2073 proc_num_ns[iproc] = 0;
2074 ns_cntr[iproc] = 0;
2075 ns_on_proc[iproc] = false;
2076 }
2077
2078 size_t iss_size = sizeof(INT);
2079 if (num_df_in_nsets[i] > 0) {
2080 iss_size += sizeof(T);
2081 }
2082
2083 find_message_info(iss_size, num_nodes_in_node_set[i], &num_node_per_message, &num_messages,
2084 &num_left_over);
2085
2086 if (Debug_Flag > 1) {
2087 fmt::print("\nMessage summary for Node Set number {}, with an ID of {}:\n", i,
2088 Node_Set_Ids[i]);
2089 fmt::print("\tNumber of messages need for node set = {}\n", num_messages);
2090 fmt::print("\tNumber of node IDs and dist. factors per message = {}\n",
2091 num_node_per_message);
2092 fmt::print("\tLength of each message = {}\n", num_node_per_message * iss_size);
2093 }
2094
2095 /* pointers into 'node_set' for the nodes that are common to
2096 both 'node_set' and the internal and border nodes for the
2097 current processor */
2098
2099 std::vector<INT> node_set(num_node_per_message);
2100 std::vector<T> node_set_df;
2101 if (num_df_in_nsets[i] > 0) {
2102 node_set_df.resize(num_node_per_message);
2103 }
2104
2105 /*---------------------------------------------------------------------*/
2106 /* LOOP OVER THE MESSAGES */
2107 /*---------------------------------------------------------------------*/
2108
2109 for (size_t imess = 0; imess < num_messages; imess++) {
2110
2111 std::vector<INT> proc_ns_node_count(Proc_Info[2]); // Count nodes on this processor.
2112
2113 size_t istart_ns = imess * num_node_per_message;
2114 if (num_left_over != 0 && imess == num_messages - 1) {
2115 num_node_per_message = num_left_over;
2116 }
2117
2118 /* Read in the part of the node set that will fit in the message */
2119 check_exodus_error(ex_get_partial_set(exoid, EX_NODE_SET, Node_Set_Ids[i], (istart_ns + 1),
2120 num_node_per_message, Data(node_set), nullptr),
2121 "ex_get_partial_set");
2122
2123 if (num_df_in_nsets[i] > 0) {
2124 check_exodus_error(ex_get_partial_set_dist_fact(exoid, EX_NODE_SET, Node_Set_Ids[i],
2125 (istart_ns + 1), num_node_per_message,
2126 Data(node_set_df)),
2127 "ex_get_partial_node_set_df");
2128 }
2129
2130 /* Renumber nodes to start at node '0' instead of node '1' */
2131 for (size_t j = 0; j < num_node_per_message; j++) {
2132 node_set[j]--;
2133 }
2134
2135 for (size_t j = 0; j < num_node_per_message; j++) {
2136 size_t node = node_set[j];
2137 size_t beg = node_proc_index[node];
2138 size_t end = node_proc_index[node + 1];
2139 for (size_t k = beg; k < end; k++) {
2140 proc_ns_node_count[gmap[k]]++;
2141 }
2142 }
2143
2144 std::vector<std::vector<INT>> proc_ns_pointer(Proc_Info[2]);
2145 for (int iproc = 0; iproc < Proc_Info[2]; iproc++) {
2146 proc_ns_pointer[iproc].reserve(proc_ns_node_count[iproc]);
2147 }
2148
2149 for (size_t j = 0; j < num_node_per_message; j++) {
2150 size_t node = node_set[j];
2151 size_t beg = node_proc_index[node];
2152 size_t end = node_proc_index[node + 1];
2153 for (size_t k = beg; k < end; k++) {
2154 proc_ns_pointer[gmap[k]].push_back(j);
2155 }
2156 }
2157
2158 /* Loop over the number of processors being handled */
2159 for (int iproc = 0; iproc < Proc_Info[2]; iproc++) {
2160 /*
2161 * If the message node set and either the Internal node or the Border
2162 * node sets intersect, then add this intersection to the list
2163 */
2164 if (proc_ns_node_count[iproc] > 0) {
2165 ns_on_proc[iproc] = true;
2166 proc_num_ns[iproc] += proc_ns_node_count[iproc];
2167
2168 /* Allocate and store node information in a temporary vector */
2169 int nset = globals.Proc_Num_Node_Sets[iproc];
2170 proc_list[iproc][nset].resize(proc_num_ns[iproc]);
2171 proc_gmap_list[iproc][nset].resize(proc_num_ns[iproc]);
2172 if (num_df_in_nsets[i] > 0) {
2173 proc_list_df[iproc][nset].resize(proc_num_ns[iproc]);
2174 }
2175
2176 for (INT j = 0; j < proc_ns_node_count[iproc]; j++) {
2177
2178 proc_list[iproc][nset][ns_cntr[iproc]] = node_set[proc_ns_pointer[iproc][j]];
2179 proc_gmap_list[iproc][nset][ns_cntr[iproc]] = proc_ns_pointer[iproc][j] + istart_ns;
2180
2181 if (num_df_in_nsets[i] > 0) {
2182 proc_list_df[iproc][nset][ns_cntr[iproc]] = node_set_df[proc_ns_pointer[iproc][j]];
2183 }
2184
2185 ns_cntr[iproc]++;
2186 }
2187 } /* End "if(intersection)" */
2188 } /* End "for(iproc=0; iproc <Proc_Info[2]; iproc++)" */
2189 } /* End "for(imess=0; imess < num_messages; imess++)" */
2190
2191 /*
2192 * If any part of this node-set is on the processor, update the various
2193 * pointers, lengths, etc.
2194 */
2195 for (int iproc = 0; iproc < Proc_Info[2]; iproc++) {
2196 if (ns_on_proc[iproc]) {
2197 ns_proc2glob[iproc][globals.Proc_Num_Node_Sets[iproc]] = i;
2198
2199 proc_list_pointer[iproc][globals.Proc_Num_Node_Sets[iproc]] = list_length[iproc];
2200
2201 proc_list_count[iproc][globals.Proc_Num_Node_Sets[iproc]] = proc_num_ns[iproc];
2202
2203 if (num_df_in_nsets[i] > 0) {
2204 proc_list_df_count[iproc][globals.Proc_Num_Node_Sets[iproc]] = proc_num_ns[iproc];
2205 }
2206 else {
2207 proc_list_df_count[iproc][globals.Proc_Num_Node_Sets[iproc]] = 0;
2208 }
2209
2210 (globals.Proc_Num_Node_Sets[iproc])++;
2211
2212 list_length[iproc] += proc_num_ns[iproc];
2213 }
2214 }
2215 } /* END "if (num_nodes_in_node_set[i] > 0)" */
2216 } /* END "for (i = 0; i < globals.Num_Node_Set; i++)" */
2217
2218 /*--------------------------------------------------------------------------*/
2219 /* WRITE PERMANENT ARRAYS FOR NODE SET INFO */
2220 /*--------------------------------------------------------------------------*/
2221
2222 for (int iproc = 0; iproc < Proc_Info[2]; iproc++) {
2223
2224 /* Allocate Permanent Arrays for node sets in one long malloc */
2225 if (globals.Num_Node_Set > 0) {
2226
2227 /*
2228 * Note that the alloc is done based on globals.Num_Node_Set, rather than
2229 * globals.Proc_Num_Node_Sets[] due to the fact that nullptr entities are
2230 * stored on processors not having a particular node set.
2231 */
2232 globals.Proc_NS_Ids[iproc] = (INT *)array_alloc(
2233 __FILE__, __LINE__, 1,
2234 (3 * globals.Num_Node_Set + 2 * globals.Proc_Num_Node_Sets[iproc] + list_length[iproc]),
2235 sizeof(INT));
2236
2237 globals.Proc_NS_Count[iproc] = globals.Proc_NS_Ids[iproc] + globals.Num_Node_Set;
2238 globals.Proc_NS_DF_Count[iproc] = globals.Proc_NS_Count[iproc] + globals.Num_Node_Set;
2239 globals.Proc_NS_Pointers[iproc] = globals.Proc_NS_DF_Count[iproc] + globals.Num_Node_Set;
2240 globals.GNode_Sets[iproc] =
2241 globals.Proc_NS_Pointers[iproc] + globals.Proc_Num_Node_Sets[iproc];
2242 globals.Proc_NS_List[iproc] = globals.GNode_Sets[iproc] + globals.Proc_Num_Node_Sets[iproc];
2243
2244 globals.Proc_NS_GNMap_List[iproc] =
2245 (INT *)array_alloc(__FILE__, __LINE__, 1, list_length[iproc], sizeof(INT));
2246
2247 if (list_length[iproc] > 0) {
2248 globals.Proc_NS_Dist_Fact[iproc].resize(list_length[iproc]);
2249 }
2250 }
2251 else {
2252 globals.Proc_NS_Ids[iproc] = nullptr;
2253 globals.Proc_NS_Count[iproc] = nullptr;
2254 globals.Proc_NS_Pointers[iproc] = nullptr;
2255 globals.Proc_NS_List[iproc] = nullptr;
2256 }
2257
2258 /*
2259 * Fill in the permanent node set arrays which have length,
2260 * globals.Proc_Num_Node_Sets, the total number of node sets defined on the
2261 * current processor.
2262 */
2263 globals.Proc_NS_List_Length[iproc] = 0;
2264 for (int i = 0; i < globals.Proc_Num_Node_Sets[iproc]; i++) {
2265 globals.GNode_Sets[iproc][i] = ns_proc2glob[iproc][i];
2266 globals.Proc_NS_Ids[iproc][i] = Node_Set_Ids[ns_proc2glob[iproc][i]];
2267 globals.Proc_NS_Count[iproc][i] = proc_list_count[iproc][i];
2268 globals.Proc_NS_Pointers[iproc][i] = proc_list_pointer[iproc][i];
2269 globals.Proc_NS_List_Length[iproc] += globals.Proc_NS_Count[iproc][i];
2270 globals.Proc_NS_DF_Count[iproc][i] = proc_list_df_count[iproc][i];
2271 }
2272
2273 /* Construct the concatenated node list */
2274 for (int i = 0; i < globals.Proc_Num_Node_Sets[iproc]; i++) {
2275 for (INT j = 0; j < globals.Proc_NS_Count[iproc][i]; j++) {
2276 globals.Proc_NS_List[iproc][globals.Proc_NS_Pointers[iproc][i] + j] =
2277 proc_list[iproc][i][j];
2278 globals.Proc_NS_GNMap_List[iproc][globals.Proc_NS_Pointers[iproc][i] + j] =
2279 proc_gmap_list[iproc][i][j];
2280
2281 if (globals.Proc_NS_DF_Count[iproc][i] > 0) {
2282 globals.Proc_NS_Dist_Fact[iproc][globals.Proc_NS_Pointers[iproc][i] + j] =
2283 proc_list_df[iproc][i][j];
2284 }
2285 }
2286 }
2287 } /* End "for(iproc=0; iproc <Proc_Info[2]; iproc++)" */
2288
2289 } /* END of routine read_node_sets () ****************************************/
2290
2291 /*****************************************************************************/
2292 /*****************************************************************************/
2293 /*****************************************************************************/
2294
2295 template <typename T, typename INT>
2296 void NemSpread<T, INT>::read_side_sets(int exoid, std::vector<INT> &num_elem_in_ssets,
2297 std::vector<INT> &num_df_in_ssets)
2298 {
2299 /*
2300 * Function which reads the side sets information from an EXODUS database
2301 * for a given processor. It then broadcasts all information to every
2302 * processor. Each processor extracts and keeps only the information that it
2303 * needs to run its local problem.
2304 *
2305 * Global Variables which are set by this routine
2306 * -----------------------------------------------
2307 *
2308 * globals.Proc_Num_Side_Sets = Number of side sets on the current processor
2309 *
2310 * globals.GSide_Sets [globals.Proc_Num_Side_Sets]
2311 * = Mapping between the local side set number and the
2312 * global side set number.
2313 *
2314 * globals.S_Ids [globals.Proc_Num_Side_Sets]
2315 * = Set IDs for the side sets defined on the current
2316 * processor
2317 *
2318 * globals.Proc_SS_Elem_Count [globals.Proc_Num_Side_Sets]
2319 * = Number of elements in the side set for the
2320 * current processor
2321 *
2322 * globals.S_Side_Count [globals.Proc_Num_Side_Sets]
2323 * = Number of nodes in the node set list for the side
2324 * set for the current processor.
2325 *
2326 * globals.Proc_SS_Elem_Pointers [globals.Proc_Num_Side_Sets]
2327 * = Side sets pointer record for elements for
2328 * the side sets in a given processor
2329 *
2330 * globals.Proc_SS_Elem_List_Length
2331 * = Length of the element list length for side sets
2332 * on the current processor
2333 *
2334 * globals.Proc_SS_Elem_List [globals.Proc_SS_Elem_List_Length]
2335 * = Concatenated vector of elements that comprise
2336 * the side set definitions on the current
2337 * processor.
2338 *
2339 * globals.S_Side_List [globals.Proc_SS_Elem_List_Length]
2340 * = Concatenated vector of sides that comprise
2341 * the side set definitions on the current
2342 * processor.
2343 */
2344
2345 /* Vector of pointers to side-set element lists. There is one pointer for each side-set */
2346 std::vector<std::vector<std::vector<INT>>> proc_elem_list(Proc_Info[2]);
2347
2348 /* Vector of pointers to side-set side lists. There is one pointer for each side-set */
2349 std::vector<std::vector<std::vector<INT>>> proc_side_list(Proc_Info[2]);
2350
2351 /* Vector of pointers to side-set global sideset element map lists. There is one pointer for each
2352 * side-set */
2353 std::vector<std::vector<std::vector<INT>>> proc_gmap_list(Proc_Info[2]);
2354
2355 /* Vector of pointers to side-set df pointers There is one pointer for each side-set */
2356 std::vector<std::vector<std::vector<INT>>> proc_df_ptr(Proc_Info[2]);
2357 std::vector<std::vector<std::vector<INT>>> proc_df_indx(Proc_Info[2]);
2358 std::vector<std::vector<std::vector<INT>>> proc_df_indx_cnt(Proc_Info[2]);
2359
2360 std::vector<std::vector<std::vector<T>>> proc_ss_df(Proc_Info[2]);
2361
2362 std::vector<INT> ss_elem_cntr(Proc_Info[2]);
2363 std::vector<bool> ss_on_proc(Proc_Info[2]); /* Flag to indicate that the ss is on the proc */
2364 std::vector<INT> elem_list_length(Proc_Info[2]); /* length of the element side-set list for the
2365 current processor */
2366 std::vector<INT> proc_num_sides_ss(Proc_Info[2]); /* Number of sides in the current side set that
2367 exist for the current processor */
2368 std::vector<INT> ntotal(Proc_Info[2]);
2369 std::vector<INT> ss_num(Proc_Info[2]);
2370
2371 /* Allocate temporary arrays */
2372 /* Array of counts of the dist. factors. */
2373 std::vector<std::vector<INT>> proc_ss_df_cnt(Proc_Info[2]);
2374
2375 /* mapping from the processor side-set numbers to the global side-set numbers */
2376 std::vector<std::vector<INT>> ss_proc2glob(Proc_Info[2]);
2377
2378 /* pointers into the concatenated element list which locate the start of the element lists */
2379 std::vector<std::vector<INT>> proc_elem_list_ptr(Proc_Info[2]);
2380
2381 /* the number of elements in each processor's element set */
2382 std::vector<std::vector<INT>> proc_elem_list_cnt(Proc_Info[2]);
2383
2384 for (int iproc = 0; iproc < Proc_Info[2]; iproc++) {
2385 proc_elem_list[iproc].resize(globals.Num_Side_Set);
2386 proc_side_list[iproc].resize(globals.Num_Side_Set);
2387 proc_df_ptr[iproc].resize(globals.Num_Side_Set);
2388 proc_df_indx[iproc].resize(globals.Num_Side_Set);
2389 proc_df_indx_cnt[iproc].resize(globals.Num_Side_Set);
2390 proc_gmap_list[iproc].resize(globals.Num_Side_Set);
2391
2392 proc_ss_df[iproc].resize(globals.Num_Side_Set);
2393
2394 ss_proc2glob[iproc].resize(globals.Num_Side_Set);
2395 proc_elem_list_ptr[iproc].resize(globals.Num_Side_Set);
2396 proc_elem_list_cnt[iproc].resize(globals.Num_Side_Set);
2397 proc_ss_df_cnt[iproc].resize(globals.Num_Side_Set);
2398 }
2399
2400 /* Create a map from 'global element id' to processor */
2401 std::vector<INT> gmap(globals.Num_Elem, -1);
2402
2403 /* Fill it in.... */
2404 for (int iproc = 0; iproc < Proc_Info[2]; iproc++) {
2405 size_t size = globals.Num_Internal_Elems[iproc] + globals.Num_Border_Elems[iproc];
2406 for (size_t i = 0; i < size; i++) {
2407 size_t ind = globals.GElems[iproc][i];
2408 gmap[ind] = iproc;
2409 }
2410 }
2411
2412 /*-------------------------------------------------------------------------*/
2413 /* LOOP OVER THE SIDE SETS */
2414 /*-------------------------------------------------------------------------*/
2415 for (int i = 0; i < globals.Num_Side_Set; i++) {
2416
2417 if (num_elem_in_ssets[i] > 0) {
2418
2419 size_t ilast = 0;
2420 int ilast_side = 0;
2421
2422 for (int iproc = 0; iproc < Proc_Info[2]; iproc++) {
2423 ss_on_proc[iproc] = false;
2424 proc_num_sides_ss[iproc] = 0;
2425 ss_elem_cntr[iproc] = 0;
2426 }
2427
2428 /* One element ID + One side ID */
2429 size_t iss_size = 3 * sizeof(INT);
2430
2431 size_t num_messages;
2432 size_t num_left_over;
2433 size_t num_elem_per_message;
2434 find_message_info(iss_size, num_elem_in_ssets[i], &num_elem_per_message, &num_messages,
2435 &num_left_over);
2436
2437 if (Debug_Flag >= 2) {
2438 fmt::print("Message summary for Side Set number {}, with an ID of {}:\n", i,
2439 Side_Set_Ids[i]);
2440 fmt::print("\tNumber of messages needed for element and side list = {}\n", num_messages);
2441 fmt::print("\tNumber of element and side IDs per message = {}\n", num_elem_per_message);
2442 fmt::print("\tLength of each message = {}\n", iss_size * num_elem_per_message);
2443 }
2444
2445 /* Allocate temporary storage for the current message for the
2446 * current side set on each proc
2447 */
2448 std::vector<INT> ss_elem_list(num_elem_per_message); /* side-set element list */
2449 std::vector<INT> ss_side_list(num_elem_per_message); /* side-set node list */
2450 std::vector<INT> ss_df_ptr(num_elem_per_message); /* pointer into the dist. factor list */
2451
2452 /*---------------------------------------------------------------------*/
2453 /* LOOP OVER THE MESSAGES */
2454 /*---------------------------------------------------------------------*/
2455 for (size_t imess = 0; imess < num_messages; imess++) {
2456
2457 if (Debug_Flag >= 2) {
2458 fmt::print("\tside set message: {} of {}\n", imess + 1, num_messages);
2459 }
2460
2461 size_t istart_ss = imess * num_elem_per_message;
2462
2463 if (num_left_over != 0 && imess == num_messages - 1) {
2464 num_elem_per_message = num_left_over;
2465 }
2466
2467 /* Read in the part of the side set that will fit in the message. */
2468
2469 check_exodus_error(ex_get_partial_set(exoid, EX_SIDE_SET, Side_Set_Ids[i], (istart_ss + 1),
2470 num_elem_per_message, Data(ss_elem_list),
2471 Data(ss_side_list)),
2472 "ex_get_partial_set");
2473
2474 /* Fill in the distribution factor pointer vector */
2475 if (imess == 0) {
2476 ilast = 0;
2477 ilast_side = 0;
2478 }
2479
2480 ss_df_ptr[0] = ilast + ilast_side;
2481
2482 for (size_t j = 1; j < num_elem_per_message; j++) {
2483
2484 /*
2485 * Kluge for "special" sidesest which has a single distribution
2486 * factor for each element instead of 1 per face node..
2487 */
2488 if (num_elem_in_ssets[i] == num_df_in_ssets[i]) {
2489 ilast_side = 1;
2490 }
2491 else {
2492
2493 ilast_side =
2494 elem_info(NN_SIDE, GM_Elem_Types[(ss_elem_list[j - 1]) - 1], ss_side_list[j - 1]);
2495
2496 /*
2497 * kludge for HEXSHELL's
2498 * where distribution factors are concerned, only count
2499 * 4 nodes on the 6 node faces (they are just like HEX's)
2500 */
2501 if (GM_Elem_Types[(ss_elem_list[j - 1]) - 1] == HEXSHELL) {
2502 ilast_side = 4;
2503 }
2504 }
2505
2506 ss_df_ptr[j] = ss_df_ptr[j - 1] + ilast_side;
2507 ilast = ss_df_ptr[j];
2508 }
2509
2510 /* Renumber elements to start at '0' instead of '1' */
2511 for (size_t j = 0; j < num_elem_per_message; ss_elem_list[j++]--) {
2512 ;
2513 }
2514
2515 #if 1
2516 /* This method may be slightly faster, but uses lots of push backs
2517 that may be bad in the long wrong. Keep both sets of code in case
2518 this turns out to be bad... The other branch is also faster than
2519 the older code, so it isn't bad to turn it on instead of this... */
2520
2521 /* Map the ss_elem_list elements to the processor that they are on */
2522 /* Note that an element may be in ss_elem_list multiple times */
2523 for (size_t j = 0; j < num_elem_per_message; j++) {
2524 int iproc = gmap[ss_elem_list[j]];
2525 int sset = globals.Proc_Num_Side_Sets[iproc];
2526 proc_elem_list[iproc][sset].push_back(ss_elem_list[j]);
2527 proc_side_list[iproc][sset].push_back(ss_side_list[j]);
2528 proc_gmap_list[iproc][sset].push_back(j + istart_ss);
2529 proc_df_ptr[iproc][sset].push_back(ss_df_ptr[j]);
2530 }
2531
2532 for (int iproc = 0; iproc < Proc_Info[2]; iproc++) {
2533 if (!proc_elem_list[iproc][globals.Proc_Num_Side_Sets[iproc]].empty()) {
2534 ss_on_proc[iproc] = true;
2535 proc_num_sides_ss[iproc] =
2536 proc_elem_list[iproc][globals.Proc_Num_Side_Sets[iproc]].size();
2537 }
2538 } /* End "for(iproc=0; iproc <Proc_Info[2]; iproc++)" */
2539 #else
2540 /* Map the ss_elem_list elements to the processor that they are on */
2541 /* Note that an element may be in ss_elem_list multiple times */
2542
2543 /* pointers into the global element side-set list for elements
2544 which are common to the processor and the side-set */
2545 std::vector<std::vector<INT>> proc_es_pointer(Proc_Info[2]);
2546
2547 for (size_t j = 0; j < num_elem_per_message; j++) {
2548 int iproc = gmap[ss_elem_list[j]];
2549 int sset = globals.Proc_Num_Side_Sets[iproc];
2550 proc_es_pointer[iproc].push_back(j);
2551 }
2552
2553 for (int iproc = 0; iproc < Proc_Info[2]; iproc++) {
2554
2555 /*
2556 * Find the intersection between the elements in the side set and the
2557 * elements owned by the current processor.
2558 */
2559
2560 if (!proc_es_pointer[iproc].empty()) {
2561 proc_num_sides_ss[iproc] += proc_es_pointer[iproc].size();
2562 ss_on_proc[iproc] = true;
2563
2564 /*
2565 * Store the information for the current side set in temporary arrays.
2566 *
2567 * This part of the routine only gets executed if the side set is
2568 * defined on the current processor.
2569 */
2570
2571 /* Allocate and store element information in a temporary vector */
2572 int sset = globals.Proc_Num_Side_Sets[iproc];
2573 proc_elem_list[iproc][sset].resize(proc_num_sides_ss[iproc]);
2574 proc_side_list[iproc][sset].resize(proc_num_sides_ss[iproc]);
2575 proc_gmap_list[iproc][sset].resize(proc_num_sides_ss[iproc]);
2576 proc_df_ptr[iproc][sset].resize(proc_num_sides_ss[iproc]);
2577
2578 /* Transfer the element, side number, and df pointers into "permanent" storage... */
2579 for (size_t j = 0; j < proc_es_pointer[iproc].size(); j++) {
2580 proc_elem_list[iproc][sset][ss_elem_cntr[iproc]] =
2581 ss_elem_list[proc_es_pointer[iproc][j]];
2582 proc_side_list[iproc][sset][ss_elem_cntr[iproc]] =
2583 ss_side_list[proc_es_pointer[iproc][j]];
2584 proc_gmap_list[iproc][sset][ss_elem_cntr[iproc]] =
2585 proc_es_pointer[iproc][j] + istart_ss;
2586 proc_df_ptr[iproc][sset][(ss_elem_cntr[iproc])++] =
2587 ss_df_ptr[proc_es_pointer[iproc][j]];
2588 }
2589 } /* End "if(ipos_elem > 0)" */
2590 } /* End "for(iproc=0; iproc <Proc_Info[2]; iproc++)" */
2591 #endif
2592 } /* End "for(imess=0; imess < num_messages; imess++)" */
2593
2594 /* Entire sideset has been read at this point */
2595 /*
2596 * If any part of this side-set is on the processor, update the various
2597 * pointers, lengths, etc.
2598 */
2599
2600 for (int iproc = 0; iproc < Proc_Info[2]; iproc++) {
2601 if (ss_on_proc[iproc]) {
2602 ss_proc2glob[iproc][globals.Proc_Num_Side_Sets[iproc]] = i;
2603
2604 proc_elem_list_ptr[iproc][globals.Proc_Num_Side_Sets[iproc]] = elem_list_length[iproc];
2605
2606 proc_elem_list_cnt[iproc][globals.Proc_Num_Side_Sets[iproc]] = proc_num_sides_ss[iproc];
2607
2608 (globals.Proc_Num_Side_Sets[iproc])++;
2609 elem_list_length[iproc] += proc_num_sides_ss[iproc];
2610 }
2611 }
2612
2613 /* Process any distribution factors in the side set */
2614 if (num_df_in_ssets[i] > 0) {
2615
2616 iss_size = sizeof(T);
2617 find_message_info(iss_size, num_df_in_ssets[i], &num_elem_per_message, &num_messages,
2618 &num_left_over);
2619
2620 if (Debug_Flag >= 4) {
2621 fmt::print("Message summary for Side Set number {}, with ID of {}:\n", i,
2622 Side_Set_Ids[i]);
2623 fmt::print("\tNumber of messages needed for distribution "
2624 "factors = {}\n",
2625 num_messages);
2626 fmt::print("\tNumber of dist. factors in each message = {}\n", num_elem_per_message);
2627 fmt::print("\tLength of each message = {}\n", (num_elem_per_message * sizeof(T)));
2628 }
2629
2630 std::vector<T> ss_dist_fact(num_elem_per_message); /* side-set distribution factors */
2631
2632 /* set up an array to help speed up the searches below */
2633 std::vector<std::vector<INT>> proc_df_map(Proc_Info[2]);
2634
2635 /* Loop over the messages needed for the distribution factors */
2636 for (size_t imess = 0; imess < num_messages; imess++) {
2637
2638 size_t istart_ss = imess * num_elem_per_message;
2639 if (num_left_over != 0 && imess == (num_messages - 1)) {
2640 num_elem_per_message = num_left_over;
2641 }
2642
2643 /* Read in the part of the side set df's that will fit in the msg. */
2644 check_exodus_error(ex_get_partial_set_dist_fact(exoid, EX_SIDE_SET, Side_Set_Ids[i],
2645 (istart_ss + 1), num_elem_per_message,
2646 Data(ss_dist_fact)),
2647 "ex_get_partial_set_dist_fact");
2648
2649 /*
2650 * At this point a processor has the list of global element IDs
2651 * that it owns that are contained in the side set. It also has
2652 * a vector of pointers into the distribution factor vector telling
2653 * where the distribution factors for a particular element in
2654 * the side set begin.
2655 */
2656
2657 /*
2658 * First figure out how many df's there are for this side set
2659 * on this processor.
2660 */
2661 for (int iproc = 0; iproc < Proc_Info[2]; iproc++) {
2662
2663 if (imess == 0) {
2664 ntotal[iproc] = 0;
2665 }
2666
2667 if (imess == 0 && ss_on_proc[iproc]) {
2668
2669 ss_num[iproc] = globals.Proc_Num_Side_Sets[iproc];
2670 ntotal[iproc] = 0;
2671
2672 size_t indx = 0;
2673 int ntotal_s = 0; // Nodes per element side...
2674
2675 size_t cnt = proc_elem_list_cnt[iproc][ss_num[iproc] - 1];
2676 proc_df_map[iproc].resize(cnt);
2677 proc_df_indx[iproc][ss_num[iproc] - 1].resize(cnt);
2678 proc_df_indx_cnt[iproc][ss_num[iproc] - 1].resize(cnt);
2679
2680 for (INT j = 0; j < proc_elem_list_cnt[iproc][ss_num[iproc] - 1]; j++) {
2681
2682 size_t loc_elem = proc_elem_list[iproc][ss_num[iproc] - 1][j];
2683 size_t loc_side = proc_side_list[iproc][ss_num[iproc] - 1][j];
2684
2685 /*
2686 * Kluge for "special" sidesest which has a single distribution
2687 * factor for each element instead of 1 per face node..
2688 */
2689 if (num_elem_in_ssets[i] == num_df_in_ssets[i]) {
2690 ntotal_s = 1;
2691 }
2692 else {
2693
2694 ntotal_s = elem_info(NN_SIDE, GM_Elem_Types[loc_elem], loc_side);
2695
2696 /*
2697 * kludge for HEXSHELL's
2698 * where distribution factors are concerned, only count
2699 * 4 nodes on the 6 node faces (they are just like HEX's)
2700 */
2701 if (GM_Elem_Types[loc_elem] == HEXSHELL) {
2702 ntotal_s = 4;
2703 }
2704 }
2705
2706 ntotal[iproc] += ntotal_s;
2707
2708 proc_df_indx[iproc][ss_num[iproc] - 1][j] = indx;
2709 proc_df_indx_cnt[iproc][ss_num[iproc] - 1][j] = 0;
2710
2711 indx += ntotal_s;
2712
2713 /* and set up map for sort in a minute */
2714 proc_df_map[iproc][j] = j;
2715 }
2716
2717 /*
2718 * now sort the proc_df_ptr array so that it is monotonic,
2719 * and can be searched more easily
2720 */
2721 my_sort(proc_elem_list_cnt[iproc][ss_num[iproc] - 1],
2722 &proc_df_ptr[iproc][ss_num[iproc] - 1][0], &proc_df_map[iproc][0]);
2723 }
2724
2725 /* Allocate memory for the dfs */
2726 if (ntotal[iproc] > 0) {
2727
2728 if (imess == 0) {
2729 proc_ss_df[iproc][ss_num[iproc] - 1].resize(ntotal[iproc]);
2730 }
2731
2732 INT k = 0;
2733 INT ss_indx = ss_num[iproc] - 1;
2734
2735 for (size_t j = 0; j < num_elem_per_message; j++) {
2736
2737 INT indx = istart_ss + j;
2738
2739 /*
2740 * Find out if this index fits in the list of this
2741 * processors indices.
2742 */
2743
2744 /* 'istart_ss' <= indx <= 'istart_ss+num_elem_per_message' */
2745 /* 'indx' increments by 1 each time through loop */
2746
2747 /* 'proc_df_ptr[iproc][ss_indx]' is the list
2748 and it is sorted....
2749 ... So, don't need a binary search, can just increment
2750 as we proceed through the loops...
2751 */
2752 if (indx >= proc_df_ptr[iproc][ss_indx][0]) {
2753
2754 while (k < proc_elem_list_cnt[iproc][ss_indx] - 1 &&
2755 indx >= proc_df_ptr[iproc][ss_indx][k + 1]) {
2756 k++;
2757 }
2758
2759 /* now get the proper location from the map */
2760 INT ipos_elem = proc_df_map[iproc][k];
2761
2762 INT idx_begin = proc_df_indx[iproc][ss_indx][ipos_elem];
2763 INT idx_end;
2764 if ((ipos_elem + 1) < proc_elem_list_cnt[iproc][ss_indx]) {
2765 idx_end = proc_df_indx[iproc][ss_indx][ipos_elem + 1];
2766 }
2767 else {
2768 idx_end = ntotal[iproc];
2769 }
2770
2771 INT idx_diff = idx_end - idx_begin;
2772
2773 /*
2774 * check to see if indx really fits in this spot
2775 *
2776 * need to make sure that indx > the k-th value since
2777 * bin_search_min will return 0 for anything less than
2778 * the first entry in the list
2779 */
2780 if (indx >= proc_df_ptr[iproc][ss_indx][k] &&
2781 indx < proc_df_ptr[iproc][ss_indx][k] + idx_diff) {
2782
2783 size_t df_loc = proc_df_indx[iproc][ss_indx][ipos_elem] +
2784 proc_df_indx_cnt[iproc][ss_indx][ipos_elem];
2785
2786 proc_ss_df[iproc][ss_indx][df_loc] = ss_dist_fact[indx - istart_ss];
2787
2788 proc_ss_df_cnt[iproc][ss_indx]++;
2789
2790 proc_df_indx_cnt[iproc][ss_indx][ipos_elem]++;
2791 }
2792 }
2793 } /* End "for(j=0; j < num_elem_per_message; j++)" */
2794 } /* End "if(ntotal[iproc] > 0)" */
2795 } /* End "for(iproc=0; iproc <Proc_Info[2]; iproc++)" */
2796 } /* End "for(imess=0; imess < num_messages; imess++)" */
2797 } /* End "if(num_df_in_ssets[i] > 0)" */
2798 } /* END "if (num_elem_in_ssets[i] > 0)" */
2799 } /* END "for (i = 0; i < globals.Num_Side_Set; i++)" */
2800
2801 /*-------------------------------------------------------------------------*/
2802 /*---------------- Store Structures back into a packed form ---------------*/
2803 /*-------------------------------------------------------------------------*/
2804
2805 /*
2806 * Allocate storage for permanent side set integer info vectors using one
2807 * long malloc statement.
2808 */
2809 for (int iproc = 0; iproc < Proc_Info[2]; iproc++) {
2810 if (globals.Num_Side_Set > 0) {
2811
2812 /*
2813 * Note that the alloc is done based on globals.Num_Side_Set, rather than
2814 * globals.Proc_Num_Side_Sets[] due to the fact that nullptr entities are
2815 * stored on processors not having a particular side set.
2816 */
2817 globals.Proc_SS_Ids[iproc] =
2818 (INT *)array_alloc(__FILE__, __LINE__, 1,
2819 (3 * globals.Num_Side_Set + 3 * globals.Proc_Num_Side_Sets[iproc] +
2820 2 * elem_list_length[iproc] + 1),
2821 sizeof(INT));
2822 globals.Proc_SS_Elem_Count[iproc] = globals.Proc_SS_Ids[iproc] + globals.Num_Side_Set;
2823 globals.Proc_SS_Elem_Pointers[iproc] =
2824 globals.Proc_SS_Elem_Count[iproc] + globals.Num_Side_Set;
2825 globals.Proc_SS_DF_Count[iproc] =
2826 globals.Proc_SS_Elem_Pointers[iproc] + globals.Proc_Num_Side_Sets[iproc];
2827 globals.GSide_Sets[iproc] = globals.Proc_SS_DF_Count[iproc] + globals.Num_Side_Set;
2828 globals.Proc_SS_Elem_List[iproc] =
2829 globals.GSide_Sets[iproc] + globals.Proc_Num_Side_Sets[iproc];
2830 globals.Proc_SS_Side_List[iproc] = globals.Proc_SS_Elem_List[iproc] + elem_list_length[iproc];
2831 globals.Proc_SS_DF_Pointers[iproc] =
2832 globals.Proc_SS_Side_List[iproc] + elem_list_length[iproc];
2833
2834 globals.Proc_SS_GEMap_List[iproc] =
2835 (INT *)array_alloc(__FILE__, __LINE__, 1, elem_list_length[iproc], sizeof(INT));
2836 }
2837
2838 /* Construct the side sets global array information */
2839 globals.Proc_SS_Elem_List_Length[iproc] = 0;
2840 for (int i = 0; i < globals.Proc_Num_Side_Sets[iproc]; i++) {
2841 globals.GSide_Sets[iproc][i] = ss_proc2glob[iproc][i];
2842 globals.Proc_SS_Ids[iproc][i] = Side_Set_Ids[ss_proc2glob[iproc][i]];
2843 globals.Proc_SS_Elem_Count[iproc][i] = proc_elem_list_cnt[iproc][i];
2844 globals.Proc_SS_Elem_Pointers[iproc][i] = proc_elem_list_ptr[iproc][i];
2845 globals.Proc_SS_Elem_List_Length[iproc] += globals.Proc_SS_Elem_Count[iproc][i];
2846 globals.Proc_SS_DF_Count[iproc][i] = proc_ss_df_cnt[iproc][i];
2847
2848 if (i == 0) {
2849 globals.Proc_SS_DF_Pointers[iproc][i] = 0;
2850 }
2851 else {
2852 globals.Proc_SS_DF_Pointers[iproc][i] =
2853 globals.Proc_SS_DF_Pointers[iproc][i - 1] + proc_ss_df_cnt[iproc][i - 1];
2854 }
2855 }
2856
2857 if (globals.Proc_Num_Side_Sets[iproc] > 0) {
2858 int i = globals.Proc_Num_Side_Sets[iproc];
2859 globals.Proc_SS_DF_Pointers[iproc][i] =
2860 globals.Proc_SS_DF_Pointers[iproc][i - 1] + proc_ss_df_cnt[iproc][i - 1];
2861
2862 if (globals.Proc_SS_DF_Pointers[iproc][i] > 0) {
2863 globals.Proc_SS_Dist_Fact[iproc].resize(globals.Proc_SS_DF_Pointers[iproc][i]);
2864 }
2865 }
2866
2867 /* Construct the concatenated element and side list */
2868 for (int i = 0; i < globals.Proc_Num_Side_Sets[iproc]; i++) {
2869 for (INT j = 0; j < globals.Proc_SS_Elem_Count[iproc][i]; j++) {
2870 globals.Proc_SS_Elem_List[iproc][globals.Proc_SS_Elem_Pointers[iproc][i] + j] =
2871 proc_elem_list[iproc][i][j];
2872 globals.Proc_SS_Side_List[iproc][globals.Proc_SS_Elem_Pointers[iproc][i] + j] =
2873 proc_side_list[iproc][i][j];
2874 globals.Proc_SS_GEMap_List[iproc][globals.Proc_SS_Elem_Pointers[iproc][i] + j] =
2875 proc_gmap_list[iproc][i][j];
2876 }
2877 for (INT j = 0; j < globals.Proc_SS_DF_Count[iproc][i]; j++) {
2878 globals.Proc_SS_Dist_Fact[iproc][globals.Proc_SS_DF_Pointers[iproc][i] + j] =
2879 proc_ss_df[iproc][i][j];
2880 }
2881 }
2882 } /* End "for(iproc)" */
2883 } /* END of routine read_side_sets */