Debian Codegraph @

packages/seacas/applications/nem_spread/pe_write_parExo_info.C

raw
1 /*
2 * Copyright(C) 1999-2024 National Technology & Engineering Solutions
3 * of Sandia, LLC (NTESS). Under the terms of Contract DE-NA0003525 with
4 * NTESS, the U.S. Government retains certain rights in this software.
5 *
6 * See packages/seacas/LICENSE for details
7 */
8 #include "copy_string_cpp.h"
9 #include "exodusII.h" // for ex_close, etc
10 #include "fmt/chrono.h"
11 #include "fmt/ostream.h"
12 #include "nem_spread.h" // for NemSpread, second, etc
13 #include "pe_common.h" // for PEX_MAX
14 #include "ps_pario_const.h" // for PIO_Time_Array
15 #include "rf_allo.h" // for safe_free, array_alloc
16 #include "rf_io_const.h" // for Debug_Flag
17 #include "sort_utils.h" // for gds_iqsort
18 #include "vector_data.h"
19 #include <cassert> // for assert
20 #include <cstddef> // for size_t
21 #include <cstdio> // for nullptr, etc
22 #include <cstdlib> // for exit, free, malloc
23 #include <cstring> // for strlen, memset, etc
24 #include <ctime> // for asctime, localtime, time, etc
25 #include <numeric>
26 #include <vector> // for vector
27 template <typename INT> struct ELEM_COMM_MAP;
28 template <typename INT> struct NODE_COMM_MAP;
29
30 #if (__cplusplus >= 201703L)
31 #define FALL_THROUGH [[fallthrough]]
32 #elif defined(__GNUC__) && __GNUC__ >= 7 && !__INTEL_COMPILER
33 #define FALL_THROUGH [[gnu::fallthrough]]
34 #else
35 #define FALL_THROUGH ((void)0)
36 #endif
37
38 namespace {
39 template <typename INT>
40 void reverse_map(INT *global, int p01, size_t gsize, INT *glmap, INT *index, INT *mapout);
41
42 template <typename INT>
43 void reverse_map(const std::vector<INT> &global, int p01, size_t gsize,
44 const std::vector<INT> &glmap, INT *index, std::vector<INT> &mapout);
45 } // namespace
46 /*
47 * need this variable for the 0 processor to hold on to the correct
48 * Exodus II database title
49 */
50 extern std::string GeomTitle;
51
52 /****************************************************************************/
53 /* This function writes parallel specific mesh information out to the */
54 /* parallel disk(s) for each processor in the salsa run. */
55 /* */
56 /* Author(s): Gary L. Hennigan (1421) */
57 /*--------------------------------------------------------------------------*/
58 /* Revision History */
59 /* */
60 /* Gary Hennigan: */
61 /* 11 November 1993 - Date of first working version on the nCube. */
62 /****************************************************************************/
63
64 template void NemSpread<float, int>::write_parExo_data(int mesh_exoid, int max_name_length,
65 int iproc, std::vector<int> &Num_Nodes_In_NS,
66 std::vector<int> &Num_Elems_In_SS,
67 std::vector<int> &Num_Elems_In_EB);
68
69 template void NemSpread<double, int>::write_parExo_data(int mesh_exoid, int max_name_length,
70 int iproc,
71 std::vector<int> &Num_Nodes_In_NS,
72 std::vector<int> &Num_Elems_In_SS,
73 std::vector<int> &Num_Elems_In_EB);
74
75 template void NemSpread<double, int64_t>::write_parExo_data(int mesh_exoid, int max_name_length,
76 int iproc,
77 std::vector<int64_t> &Num_Nodes_In_NS,
78 std::vector<int64_t> &Num_Elems_In_SS,
79 std::vector<int64_t> &Num_Elems_In_EB);
80
81 template void NemSpread<float, int64_t>::write_parExo_data(int mesh_exoid, int max_name_length,
82 int iproc,
83 std::vector<int64_t> &Num_Nodes_In_NS,
84 std::vector<int64_t> &Num_Elems_In_SS,
85 std::vector<int64_t> &Num_Elems_In_EB);
86
87 template <typename T, typename INT>
88 void NemSpread<T, INT>::write_parExo_data(int mesh_exoid, int max_name_length, int iproc,
89 std::vector<INT> &Num_Nodes_In_NS,
90 std::vector<INT> &Num_Elems_In_SS,
91 std::vector<INT> &Num_Elems_In_EB)
92 {
93 /* Performance metrics. */
94 size_t bytes_out = 0;
95 double total_out_time = 0.0;
96 double tt1;
97
98 int error;
99 /****************************BEGIN EXECUTION*********************************/
100
101 /*+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++*/
102 /* PARALLEL EXODUSII SECTION */
103 /*+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++*/
104
105 int proc_for = Proc_Ids[iproc];
106 int num_proc_for = Proc_Info[0];
107
108 size_t itotal_nodes = globals.Num_Internal_Nodes[iproc] + globals.Num_Border_Nodes[iproc] +
109 globals.Num_External_Nodes[iproc];
110 size_t itotal_elems = globals.Num_Internal_Elems[iproc] + globals.Num_Border_Elems[iproc];
111
112 bytes_out += 5 * sizeof(INT);
113 tt1 = second();
114
115 if (Debug_Flag >= 4) {
116 fmt::print("Putting init global info in file id: {}\n", mesh_exoid);
117 fmt::print("\tNumber Global Nodes: {}\n", globals.Num_Node);
118 fmt::print("\tNumber Global Elements: {}\n", globals.Num_Elem);
119 fmt::print("\tNumber Global Element Blocks: {}\n", globals.Num_Elem_Blk);
120 fmt::print("\tNumber Global Node Sets: {}\n", globals.Num_Node_Set);
121 fmt::print("\tNumber Global Side Sets: {}\n", globals.Num_Side_Set);
122 }
123
124 if (ex_put_init_global(mesh_exoid, globals.Num_Node, globals.Num_Elem, globals.Num_Elem_Blk,
125 globals.Num_Node_Set, globals.Num_Side_Set) < 0) {
126 fmt::print(stderr,
127 "[{}]: ERROR, Unable to put global initial "
128 "information in parallel mesh file!\n",
129 __func__);
130 exit(1);
131 }
132
133 PIO_Time_Array[0] = (second() - tt1);
134 total_out_time += PIO_Time_Array[0];
135
136 /*
137 * Find the number of distinct nodal communication maps that will
138 * be needed. This assumes that there is only a single input
139 * nodal communication map that stores information for all
140 * processors.
141 */
142 int ncomm_cnt = 0;
143 std::vector<NODE_COMM_MAP<INT>> n_comm_map;
144 std::vector<INT> n_comm_ids;
145 std::vector<INT> n_comm_ncnts;
146
147 if (globals.Num_N_Comm_Maps[iproc] > 0 && globals.N_Comm_Map[iproc].node_cnt != 0) {
148 ncomm_cnt = 1;
149 ex_entity_id itemp = globals.N_Comm_Map[iproc].proc_ids[0];
150
151 /* First find the count */
152 for (size_t i1 = 1; i1 < globals.N_Comm_Map[iproc].node_cnt; i1++) {
153 if (globals.N_Comm_Map[iproc].proc_ids[i1] != itemp) {
154 ncomm_cnt++;
155 itemp = globals.N_Comm_Map[iproc].proc_ids[i1];
156 }
157 }
158
159 /* Allocate memory for the nodal communication maps */
160 n_comm_map.resize(ncomm_cnt);
161 n_comm_ids.resize(ncomm_cnt);
162 n_comm_ncnts.resize(ncomm_cnt);
163
164 /* Find the size of each map */
165 ncomm_cnt = 0;
166 itemp = globals.N_Comm_Map[iproc].proc_ids[0];
167 n_comm_map[0].node_cnt = 1;
168 for (size_t i1 = 1; i1 < globals.N_Comm_Map[iproc].node_cnt; i1++) {
169 if (globals.N_Comm_Map[iproc].proc_ids[i1] != itemp) {
170 itemp = globals.N_Comm_Map[iproc].proc_ids[i1];
171 ncomm_cnt++;
172 n_comm_map[ncomm_cnt].node_cnt = 1;
173 }
174 else {
175 (n_comm_map[ncomm_cnt].node_cnt)++;
176 }
177 }
178
179 ncomm_cnt++;
180 size_t temp = 0;
181
182 /* Allocate memory for the maps */
183 for (int i1 = 0; i1 < ncomm_cnt; i1++) {
184 n_comm_map[i1].proc_ids.resize(n_comm_map[i1].node_cnt);
185 n_comm_map[i1].node_ids.resize(n_comm_map[i1].node_cnt);
186
187 for (size_t i2 = 0; i2 < n_comm_map[i1].node_cnt; i2++) {
188 n_comm_map[i1].proc_ids[i2] = globals.N_Comm_Map[iproc].proc_ids[temp];
189 n_comm_map[i1].node_ids[i2] = globals.N_Comm_Map[iproc].node_ids[temp++];
190 }
191
192 n_comm_ncnts[i1] = n_comm_map[i1].node_cnt;
193 n_comm_ids[i1] = n_comm_map[i1].proc_ids[0];
194 }
195 }
196 else {
197 ncomm_cnt = 0;
198 }
199
200 /*
201 * Find the number of distinct elemental communication maps that will
202 * be needed. This assumes that there is only a single input
203 * elemental communication map that stores information for all
204 * processors.
205 */
206 INT ecomm_cnt = 0;
207 std::vector<INT> e_comm_ids;
208 std::vector<INT> e_comm_ecnts;
209 std::vector<ELEM_COMM_MAP<INT>> e_comm_map;
210
211 if (globals.Num_E_Comm_Maps[iproc] > 0) {
212 ecomm_cnt = 1;
213 INT itemp = globals.E_Comm_Map[iproc].proc_ids[0];
214
215 /* First find the count */
216 for (size_t i1 = 1; i1 < globals.E_Comm_Map[iproc].elem_cnt; i1++) {
217 if (globals.E_Comm_Map[iproc].proc_ids[i1] != itemp) {
218 ecomm_cnt++;
219 itemp = globals.E_Comm_Map[iproc].proc_ids[i1];
220 }
221 }
222
223 /* Allocate memory for the elemental communication maps */
224 e_comm_map.resize(ecomm_cnt);
225 e_comm_ids.resize(ecomm_cnt);
226 e_comm_ecnts.resize(ecomm_cnt);
227
228 /* Find the size of each map */
229 ecomm_cnt = 0;
230 itemp = globals.E_Comm_Map[iproc].proc_ids[0];
231 e_comm_map[0].elem_cnt = 1;
232 for (size_t i1 = 1; i1 < globals.E_Comm_Map[iproc].elem_cnt; i1++) {
233 if (globals.E_Comm_Map[iproc].proc_ids[i1] != itemp) {
234 itemp = globals.E_Comm_Map[iproc].proc_ids[i1];
235 ecomm_cnt++;
236 e_comm_map[ecomm_cnt].elem_cnt = 1;
237 }
238 else {
239 (e_comm_map[ecomm_cnt].elem_cnt)++;
240 }
241 }
242
243 ecomm_cnt++;
244 itemp = 0;
245
246 /* Allocate memory for the maps */
247 for (INT i1 = 0; i1 < ecomm_cnt; i1++) {
248 e_comm_map[i1].proc_ids.resize(e_comm_map[i1].elem_cnt);
249 e_comm_map[i1].elem_ids.resize(e_comm_map[i1].elem_cnt);
250 e_comm_map[i1].side_ids.resize(e_comm_map[i1].elem_cnt);
251
252 for (size_t i2 = 0; i2 < e_comm_map[i1].elem_cnt; i2++) {
253 e_comm_map[i1].proc_ids[i2] = globals.E_Comm_Map[iproc].proc_ids[itemp];
254 e_comm_map[i1].elem_ids[i2] = globals.E_Comm_Map[iproc].elem_ids[itemp];
255 e_comm_map[i1].side_ids[i2] = globals.E_Comm_Map[iproc].side_ids[itemp++];
256 }
257
258 e_comm_ecnts[i1] = e_comm_map[i1].elem_cnt;
259 e_comm_ids[i1] = e_comm_map[i1].proc_ids[0];
260 }
261 }
262 else {
263 ecomm_cnt = 0;
264 }
265
266 /* Output load balance information */
267 bytes_out += 9 * sizeof(INT) + 2 * sizeof(char);
268 tt1 = second();
269
270 if (Debug_Flag >= 4) {
271 fmt::print("Putting init Nemesis info in file id: {}\n", mesh_exoid);
272 fmt::print("\tNumber of Processor for: {}\n", num_proc_for);
273 }
274
275 if (ex_put_init_info(mesh_exoid, num_proc_for, 1, (char *)"p") < 0) {
276 fmt::print(stderr, "[{}]: ERROR, unable to output init info!\n", __func__);
277 exit(1);
278 }
279
280 if (Debug_Flag >= 6) {
281 fmt::print("Putting init load balance info in file id: {}\n", mesh_exoid);
282 fmt::print("\tNumber Internal Nodes: {}\n", globals.Num_Internal_Nodes[iproc]);
283 fmt::print("\tNumber Border Nodes: {}\n", globals.Num_Border_Nodes[iproc]);
284 fmt::print("\tNumber External Nodes: {}\n", globals.Num_External_Nodes[iproc]);
285 fmt::print("\tNumber Internal Elements: {}\n", globals.Num_Internal_Elems[iproc]);
286 fmt::print("\tNumber Border Elements: {}\n", globals.Num_Border_Elems[iproc]);
287 fmt::print("\tNumber Nodal Cmaps: {}\n", ncomm_cnt);
288 fmt::print("\tNumber Elemental Cmaps: {}\n", ecomm_cnt);
289 fmt::print("\tProccesor For: {}\n", proc_for);
290 }
291
292 if (ex_put_loadbal_param(mesh_exoid, globals.Num_Internal_Nodes[iproc],
293 globals.Num_Border_Nodes[iproc], globals.Num_External_Nodes[iproc],
294 globals.Num_Internal_Elems[iproc], globals.Num_Border_Elems[iproc],
295 ncomm_cnt, ecomm_cnt, proc_for) < 0) {
296 fmt::print(stderr, "[{}]: ERROR, unable to output load balance info\n", __func__);
297 ex_close(mesh_exoid);
298 exit(1);
299 }
300
301 PIO_Time_Array[1] = (second() - tt1);
302 total_out_time += PIO_Time_Array[1];
303
304 /* Output the communication map */
305 bytes_out += 4 * sizeof(INT);
306 tt1 = second();
307
308 if (ex_put_cmap_params(mesh_exoid, Data(n_comm_ids), Data(n_comm_ncnts), Data(e_comm_ids),
309 Data(e_comm_ecnts), proc_for) < 0) {
310 fmt::print(stderr, "[{}]: ERROR, unable to output comm map params!\n", __func__);
311 ex_close(mesh_exoid);
312 exit(1);
313 }
314
315 PIO_Time_Array[2] = (second() - tt1);
316 total_out_time += PIO_Time_Array[2];
317
318 /* Convert Elem_Map to local element numbering */
319 reverse_map(globals.Elem_Map[iproc], 0, itotal_elems, globals.GElems[iproc], (INT *)nullptr,
320 globals.Elem_Map[iproc]);
321 /* Convert element IDs in the comm map to local numbering */
322 for (INT i0 = 0; i0 < ecomm_cnt; i0++) {
323 reverse_map(e_comm_map[i0].elem_ids, 0, e_comm_map[i0].elem_cnt, globals.GElems[iproc],
324 (INT *)nullptr, e_comm_map[i0].elem_ids);
325 }
326
327 /* Sort the globals.GNodes array using the index array 'loc_index' */
328 std::vector<INT> loc_index(itotal_nodes);
329
330 /* Initialize index array */
331 for (size_t i2 = 0; i2 < itotal_nodes; i2++) {
332 loc_index[i2] = i2;
333 }
334
335 /*
336 * Sort the globals.GNodes[iproc] array via the index array
337 * 'loc_index'
338 */
339 gds_iqsort(Data(globals.GNodes[iproc]), Data(loc_index), itotal_nodes);
340
341 /* Convert nodal IDs in the comm map to local numbering */
342 for (INT i0 = 0; i0 < ncomm_cnt; i0++) {
343 reverse_map(n_comm_map[i0].node_ids, 0, n_comm_map[i0].node_cnt, globals.GNodes[iproc],
344 Data(loc_index), n_comm_map[i0].node_ids);
345 }
346
347 PIO_Time_Array[3] = 0.0;
348
349 if (globals.Num_N_Comm_Maps[iproc] > 0 && globals.N_Comm_Map[iproc].node_cnt != 0) {
350
351 bytes_out += 2 * globals.Num_External_Nodes[iproc] * sizeof(INT);
352 tt1 = second();
353
354 for (INT i1 = 0; i1 < ncomm_cnt; i1++) {
355 if (ex_put_node_cmap(mesh_exoid, n_comm_ids[i1], Data(n_comm_map[i1].node_ids),
356 Data(n_comm_map[i1].proc_ids), proc_for) < 0) {
357 fmt::print(stderr, "[{}]: ERROR, unable to output nodal comm map!\n", __func__);
358 ex_close(mesh_exoid);
359 exit(1);
360 }
361 n_comm_map[i1].proc_ids.clear();
362 n_comm_map[i1].node_ids.clear();
363 }
364
365 PIO_Time_Array[3] += (second() - tt1);
366 }
367 if (globals.Num_E_Comm_Maps[iproc] > 0) {
368 bytes_out += 3 * (globals.E_Comm_Map[iproc].elem_cnt) * sizeof(INT);
369 tt1 = second();
370
371 for (INT i1 = 0; i1 < ecomm_cnt; i1++) {
372 if (ex_put_elem_cmap(mesh_exoid, e_comm_ids[i1], Data(e_comm_map[i1].elem_ids),
373 Data(e_comm_map[i1].side_ids), Data(e_comm_map[i1].proc_ids),
374 proc_for) < 0) {
375 fmt::print(stderr, "[{}]: ERROR, unable to output elemental comm map!\n", __func__);
376 ex_close(mesh_exoid);
377 exit(1);
378 }
379
380 e_comm_map[i1].proc_ids.clear();
381 e_comm_map[i1].elem_ids.clear();
382 e_comm_map[i1].side_ids.clear();
383 }
384 PIO_Time_Array[3] += (second() - tt1);
385 }
386
387 total_out_time += PIO_Time_Array[3];
388
389 /* Output the global node set parameters */
390 if (globals.Num_Node_Set > 0) {
391
392 std::vector<INT> glob_ns_df_cnts(globals.Num_Node_Set);
393
394 bytes_out += 3 * globals.Num_Node_Set * sizeof(INT);
395 tt1 = second();
396
397 if (ex_put_ns_param_global(mesh_exoid, Data(Node_Set_Ids), Data(Num_Nodes_In_NS),
398 Data(glob_ns_df_cnts)) < 0) {
399 fmt::print(stderr, "[{}]: ERROR, unable to output global node-set params\n", __func__);
400 ex_close(mesh_exoid);
401 exit(1);
402 }
403
404 PIO_Time_Array[4] = (second() - tt1);
405 total_out_time += PIO_Time_Array[4];
406 }
407
408 /* Output the global side set parameters */
409 if (globals.Num_Side_Set > 0) {
410
411 std::vector<INT> glob_ss_df_cnts(globals.Num_Side_Set);
412
413 bytes_out += 3 * globals.Num_Side_Set * sizeof(INT);
414 tt1 = second();
415
416 if (ex_put_ss_param_global(mesh_exoid, Data(Side_Set_Ids), Data(Num_Elems_In_SS),
417 Data(glob_ss_df_cnts)) < 0) {
418 fmt::print(stderr, "[{}]: ERROR, unable to output global side-set params\n", __func__);
419 ex_close(mesh_exoid);
420 exit(1);
421 }
422
423 PIO_Time_Array[5] = (second() - tt1);
424 total_out_time += PIO_Time_Array[5];
425 }
426
427 bytes_out += globals.Num_Elem_Blk * sizeof(INT);
428 tt1 = second();
429
430 if (ex_put_eb_info_global(mesh_exoid, Data(Elem_Blk_Ids), Data(Num_Elems_In_EB)) < 0) {
431 fmt::print(stderr, "[{}]: ERROR, unable to output global elem blk IDs\n", __func__);
432 ex_close(mesh_exoid);
433 exit(1);
434 }
435
436 PIO_Time_Array[6] = (second() - tt1);
437 total_out_time += PIO_Time_Array[6];
438 /*+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++*/
439
440 /* Generate a QA record for the utility */
441 time_t date_time = time(nullptr);
442 auto *lt = std::localtime(&date_time);
443
444 char qa_time[MAX_STR_LENGTH + 1];
445 char qa_name[MAX_STR_LENGTH + 1];
446 char qa_vers[MAX_STR_LENGTH + 1];
447 char qa_date[MAX_STR_LENGTH + 1];
448
449 std::string time = fmt::format("{:%H:%M:%S}", *lt);
450 std::string date = fmt::format("{:%Y/%m/%d}", *lt);
451 copy_string(qa_date, date);
452 copy_string(qa_time, time);
453 copy_string(qa_name, UTIL_NAME);
454 copy_string(qa_vers, VER_STR);
455
456 if (globals.Num_QA_Recs > 0) {
457 copy_string(globals.QA_Record[(4 * (globals.Num_QA_Recs - 1)) + 0], qa_name,
458 MAX_STR_LENGTH + 1);
459 copy_string(globals.QA_Record[(4 * (globals.Num_QA_Recs - 1)) + 1], qa_vers,
460 MAX_STR_LENGTH + 1);
461 copy_string(globals.QA_Record[(4 * (globals.Num_QA_Recs - 1)) + 2], qa_date,
462 MAX_STR_LENGTH + 1);
463 copy_string(globals.QA_Record[(4 * (globals.Num_QA_Recs - 1)) + 3], qa_time,
464 MAX_STR_LENGTH + 1);
465
466 /* Output QA records to screen */
467 if (Debug_Flag >= 4) {
468 fmt::print("Number of QA records: {}\n", globals.Num_QA_Recs);
469 if (Debug_Flag >= 6) {
470 fmt::print("QA Records:\n");
471 for (INT i1 = 0; i1 < 4 * (globals.Num_QA_Recs); i1++) {
472 fmt::print("\t{}\n", globals.QA_Record[i1]);
473 }
474 }
475 }
476 }
477
478 /* Output the QA and Info records */
479 for (INT i1 = 0; i1 < 4 * globals.Num_QA_Recs; i1++) {
480 bytes_out += (MAX_STR_LENGTH + MAX_LINE_LENGTH) * sizeof(char);
481 }
482
483 tt1 = second();
484
485 if (ex_put_qa(mesh_exoid, globals.Num_QA_Recs, (char *(*)[4]) & globals.QA_Record[0]) < 0) {
486 fmt::print(stderr, "[{}]: ERROR Could not put QA records\n", __func__);
487 ex_close(mesh_exoid);
488 exit(1);
489 }
490
491 if (globals.Num_Info_Recs > 0) {
492 if (Debug_Flag >= 4) {
493 fmt::print("Number of info records: {}\n", globals.Num_Info_Recs);
494 }
495
496 if (ex_put_info(mesh_exoid, globals.Num_Info_Recs, globals.Info_Record) < 0) {
497 fmt::print(stderr, "[{}]: ERROR Could not put Info records\n", __func__);
498 ex_close(mesh_exoid);
499 exit(1);
500 }
501 }
502
503 PIO_Time_Array[8] = (second() - tt1);
504 total_out_time += PIO_Time_Array[8];
505
506 /* Output the assembly information (if any). This puts the
507 file in/out of define mode, so should be early in the write stage
508 */
509 if (Debug_Flag >= 4) {
510 fmt::print("Number of Assemblies: {}\n", globals.Num_Assemblies);
511 }
512
513 if (globals.Num_Assemblies > 0) {
514 ex_put_assemblies(mesh_exoid, globals.Assemblies.size(), Data(globals.Assemblies));
515 }
516
517 /* Output the coordinate frame information (if any). This puts the
518 file in/out of define mode, so should be early in the write stage
519 */
520 if (Debug_Flag >= 4) {
521 fmt::print("Number of Coordinate Frames: {}\n", globals.Num_Coordinate_Frames);
522 }
523
524 if (globals.Num_Coordinate_Frames > 0) {
525 T *Coordinate_Frame_Coordinates = globals.Coordinate_Frame_Coordinates;
526 if (ex_put_coordinate_frames(mesh_exoid, globals.Num_Coordinate_Frames,
527 globals.Coordinate_Frame_Ids, Coordinate_Frame_Coordinates,
528 globals.Coordinate_Frame_Tags) < 0) {
529 fmt::print(stderr, "[{}]: ERROR, Unable to put coordinate frame data in parallel mesh file\n",
530 __func__);
531 ex_close(mesh_exoid);
532 exit(1);
533 }
534 }
535
536 std::string cTitle;
537
538 if (proc_for == 0) {
539 cTitle = GeomTitle;
540 }
541 else {
542 cTitle = fmt::format("Parallel Mesh File for Processor {}", proc_for);
543 }
544
545 /* Output the initial information to the parallel Exodus file(s) */
546 bytes_out += cTitle.length() * sizeof(char) + 6 * sizeof(INT);
547 tt1 = second();
548
549 if (Debug_Flag >= 4) {
550 fmt::print("Putting init info in file id: {}\n", mesh_exoid);
551 fmt::print("\tTitle: {}\n", cTitle);
552 fmt::print("\tNumber Dimensions: {}\n", globals.Num_Dim);
553 fmt::print("\tNumber Nodes: {}\n", itotal_nodes);
554 fmt::print("\tNumber Elements: {}\n",
555 globals.Num_Internal_Elems[iproc] + globals.Num_Border_Elems[iproc]);
556 fmt::print("\tNumber Element Blocks: {}\n", globals.Num_Elem_Blk);
557 fmt::print("\tNumber Node Sets: {}\n", globals.Num_Node_Set);
558 fmt::print("\tNumber Side Sets: {}\n", globals.Num_Side_Set);
559 }
560
561 if (ex_put_init(mesh_exoid, cTitle.c_str(), globals.Num_Dim, itotal_nodes,
562 globals.Num_Internal_Elems[iproc] + globals.Num_Border_Elems[iproc],
563 globals.Num_Elem_Blk, globals.Num_Node_Set, globals.Num_Side_Set) < 0) {
564 fmt::print(stderr, "[{}]: ERROR, Unable to put initial info in parallel mesh file\n", __func__);
565 ex_close(mesh_exoid);
566 exit(1);
567 }
568
569 /* Output names... */
570 if (globals.Num_Elem_Blk > 0) {
571 error = ex_put_names(mesh_exoid, EX_ELEM_BLOCK, &Elem_Blk_Names[0]);
572 check_exodus_error(error, "ex_put_names");
573 }
574 if (globals.Num_Node_Set > 0) {
575 error = ex_put_names(mesh_exoid, EX_NODE_SET, &Node_Set_Names[0]);
576 check_exodus_error(error, "ex_put_names");
577 }
578 if (globals.Num_Side_Set > 0) {
579 error = ex_put_names(mesh_exoid, EX_SIDE_SET, &Side_Set_Names[0]);
580 check_exodus_error(error, "ex_put_names");
581 }
582
583 PIO_Time_Array[7] = (second() - tt1);
584 total_out_time += PIO_Time_Array[7];
585
586 /* Assign the coordinates to the coord_vector */
587 T *x_coord = nullptr;
588 T *y_coord = nullptr;
589 T *z_coord = nullptr;
590 if (itotal_nodes > 0) {
591 switch (globals.Num_Dim) {
592 case 3: z_coord = globals.Coor[iproc][2]; FALL_THROUGH;
593 case 2: y_coord = globals.Coor[iproc][1]; FALL_THROUGH;
594 case 1: x_coord = globals.Coor[iproc][0]; break;
595 }
596 }
597
598 /* Output the coordinates to the parallel Exodus file */
599 bytes_out += globals.Num_Dim * itotal_nodes * io_ws;
600 tt1 = second();
601
602 if (Debug_Flag >= 4) {
603 fmt::print("Putting coordinate info in file id: {}\n", mesh_exoid);
604 }
605 if (ex_put_coord(mesh_exoid, x_coord, y_coord, z_coord) < 0) {
606 fmt::print(stderr, "[{}]: ERROR, could not write out nodal coordinates\n", __func__);
607 ex_close(mesh_exoid);
608 exit(1);
609 }
610
611 PIO_Time_Array[9] = (second() - tt1);
612 total_out_time += PIO_Time_Array[9];
613
614 bytes_out += itotal_nodes * sizeof(INT);
615 tt1 = second();
616
617 /*
618 * The Nemesis node maps are lists of internal, border and external
619 * FEM node numbers. These are output as local node numbers.
620 */
621 INT *nem_node_mapi = (INT *)array_alloc(__FILE__, __LINE__, 1, itotal_nodes, sizeof(INT));
622 INT *nem_node_mapb = nem_node_mapi + globals.Num_Internal_Nodes[iproc];
623 INT *nem_node_mape = nem_node_mapb + globals.Num_Border_Nodes[iproc];
624 std::iota(nem_node_mapi, nem_node_mapi + itotal_nodes, (INT)1);
625
626 if (ex_put_processor_node_maps(mesh_exoid, nem_node_mapi, nem_node_mapb, nem_node_mape,
627 proc_for) < 0) {
628 fmt::print(stderr, "[{}]: ERROR, could not write Nemesis nodal number map!\n", __func__);
629 check_exodus_error(ex_close(mesh_exoid), "ex_close");
630 ex_close(mesh_exoid);
631 exit(1);
632 }
633
634 /* If non-nullptr, output the global node id map which preserves
635 the global node ids in the original mesh */
636 if (!globals.Proc_Global_Node_Id_Map[iproc].empty()) {
637 bytes_out += itotal_nodes * sizeof(INT);
638 if (ex_put_map_param(mesh_exoid, 1, 0) < 0) {
639 fmt::print(stderr, "[{}]: ERROR, unable to define global node map parameters!\n", __func__);
640 ex_close(mesh_exoid);
641 exit(1);
642 }
643
644 if (ex_put_num_map(mesh_exoid, EX_NODE_MAP, 1, Data(globals.Proc_Global_Node_Id_Map[iproc])) <
645 0) {
646 fmt::print(stderr, "[{}]: ERROR, unable to output global node id map!\n", __func__);
647 ex_close(mesh_exoid);
648 exit(1);
649 }
650 if (ex_put_name(mesh_exoid, EX_NODE_MAP, 1, "original_global_id_map") < 0) {
651 fmt::print(stderr, "[{}]: ERROR, unable to define global node map name!\n", __func__);
652 ex_close(mesh_exoid);
653 exit(1);
654 }
655 }
656
657 PIO_Time_Array[11] = (second() - tt1);
658 total_out_time += PIO_Time_Array[11];
659
660 safe_free((void **)&nem_node_mapi);
661
662 PIO_Time_Array[13] = 0.0;
663 PIO_Time_Array[14] = 0.0;
664 PIO_Time_Array[15] = 0.0;
665
666 /*
667 * Generate a list of the global element blocks, some of which may be
668 * nullptr on a given processor.
669 */
670 {
671 /*
672 * Start a local block so we can define some variables locally
673 * here instead of a few thousand lines up from here...
674 */
675 std::vector<INT> EB_Ids(globals.Num_Elem_Blk);
676 std::vector<INT> EB_Cnts(globals.Num_Elem_Blk);
677 std::vector<INT> EB_NperE(globals.Num_Elem_Blk);
678 std::vector<INT> EB_Nattr(globals.Num_Elem_Blk);
679
680 char **EB_Types = (char **)array_alloc(__FILE__, __LINE__, 2, globals.Num_Elem_Blk,
681 MAX_STR_LENGTH + 1, sizeof(char));
682 if (EB_Types == nullptr) {
683 fmt::print(stderr, "{}: fatal: insufficient memory\n", __func__);
684 exit(1);
685 }
686
687 for (INT i1 = 0; i1 < globals.Num_Elem_Blk; i1++) {
688 memset(EB_Types[i1], '\0', (MAX_STR_LENGTH + 1));
689 }
690
691 INT cnt = 0;
692 for (INT i1 = 0; i1 < globals.Num_Elem_Blk; i1++) {
693 bool ifound = false;
694 for (INT i2 = 0; i2 < globals.Proc_Num_Elem_Blk[iproc]; i2++) {
695 if (globals.Proc_Elem_Blk_Ids[iproc][i2] == Elem_Blk_Ids[i1]) {
696 ifound = true;
697 break;
698 }
699 }
700
701 /*
702 * If it's not found then this element block is null on the current
703 * processor.
704 */
705 if (!ifound) {
706 INT iblk = globals.Proc_Num_Elem_Blk[iproc] + cnt;
707 globals.Proc_Elem_Blk_Ids[iproc][iblk] = Elem_Blk_Ids[i1];
708 globals.Proc_Num_Elem_In_Blk[iproc][iblk] = 0;
709 globals.Proc_Nodes_Per_Elem[iproc][iblk] = 0;
710 globals.Proc_Num_Attr[iproc][iblk] = 0;
711 cnt++;
712 }
713 }
714
715 /* Output the elemental block(s). */
716 for (INT i1 = 0; i1 < globals.Num_Elem_Blk; i1++) {
717
718 ex_entity_id iglobal_blk = Elem_Blk_Ids[i1];
719
720 /* Find the local element block index */
721 INT ilocal;
722 for (ilocal = 0; ilocal < globals.Num_Elem_Blk; ilocal++) {
723 if (globals.Proc_Elem_Blk_Ids[iproc][ilocal] == iglobal_blk) {
724 break;
725 }
726 }
727
728 /* Error check */
729 if (ilocal >= globals.Num_Elem_Blk) {
730 fmt::print(stderr, "[{}]: Error finding local element block ID\n", __func__);
731 exit(1);
732 }
733
734 /* If it's a non-null block output all information */
735 EB_Ids[i1] = iglobal_blk;
736 if (ilocal < globals.Proc_Num_Elem_Blk[iproc]) {
737
738 /* Generate the ExodusII element name */
739 copy_string(EB_Types[i1], Elem_Blk_Types[i1], MAX_STR_LENGTH);
740
741 EB_Cnts[i1] = globals.Proc_Num_Elem_In_Blk[iproc][ilocal];
742 EB_NperE[i1] = globals.Proc_Nodes_Per_Elem[iproc][ilocal];
743 EB_Nattr[i1] = globals.Proc_Num_Attr[iproc][ilocal];
744 }
745 }
746 if (Debug_Flag >= 4) {
747 fmt::print("Putting concat_elem_block info in file id: {}\n", mesh_exoid);
748 }
749 error = ex_put_concat_elem_block(mesh_exoid, Data(EB_Ids), &EB_Types[0], Data(EB_Cnts),
750 Data(EB_NperE), Data(EB_Nattr), 1);
751 check_exodus_error(error, "ex_put_concat_elem_block");
752
753 safe_free(reinterpret_cast<void **>(&EB_Types));
754
755 /* Output attribute names for each element block */
756 for (INT i1 = 0; i1 < globals.Num_Elem_Blk; i1++) {
757
758 ex_entity_id iglobal_blk = Elem_Blk_Ids[i1];
759
760 /* Find the local element block index */
761 INT ilocal;
762 for (ilocal = 0; ilocal < globals.Num_Elem_Blk; ilocal++) {
763 if (globals.Proc_Elem_Blk_Ids[iproc][ilocal] == iglobal_blk) {
764 break;
765 }
766 }
767
768 /* If it's a non-null block output attribute name information */
769 if (ilocal < globals.Proc_Num_Elem_Blk[iproc]) {
770 if (globals.Proc_Num_Attr[iproc][ilocal] > 0) {
771 if (ex_put_attr_names(mesh_exoid, EX_ELEM_BLOCK, Elem_Blk_Ids[i1],
772 Elem_Blk_Attr_Names[i1]) < 0) {
773 fmt::print(stderr, "[{}]: ERROR, could not write Exodus attribute names!\n", __func__);
774 ex_close(mesh_exoid);
775 exit(1);
776 }
777 }
778 }
779 }
780
781 /* Reset globals.GNodes to start at 1 instead of 0 */
782 for (size_t i1 = 0; i1 < itotal_nodes; (globals.GNodes[iproc][i1++])++) {
783 ;
784 }
785
786 /* Output the Exodus node number map */
787 bytes_out += itotal_nodes * sizeof(INT);
788 tt1 = second();
789
790 if (Debug_Flag >= 4) {
791 fmt::print("Putting node_num_map in file id: {}\n", mesh_exoid);
792 }
793 if (ex_put_id_map(mesh_exoid, EX_NODE_MAP, Data(globals.GNodes[iproc])) < 0) {
794 fmt::print(stderr, "[{}]: ERROR, could not write Exodus node number map!\n", __func__);
795 ex_close(mesh_exoid);
796 exit(1);
797 }
798
799 PIO_Time_Array[10] = (second() - tt1);
800 total_out_time += PIO_Time_Array[10];
801
802 /*
803 * Allocate memory for the elemental map. Currently this map is assigned
804 * as a linear array since it is not really used.
805 */
806 std::vector<INT> iElem_Map(globals.Num_Internal_Elems[iproc] + globals.Num_Border_Elems[iproc]);
807 for (INT i1 = 0; i1 < globals.Num_Internal_Elems[iproc] + globals.Num_Border_Elems[iproc];
808 i1++) {
809 iElem_Map[i1] = globals.GElems[iproc][i1] + 1;
810 }
811
812 bytes_out +=
813 2 * globals.Num_Internal_Elems[iproc] * globals.Num_Border_Elems[iproc] * sizeof(INT);
814 tt1 = second();
815
816 if (Debug_Flag >= 4) {
817 fmt::print("Putting elem_num_map info in file id: {}\n", mesh_exoid);
818 }
819 if (ex_put_id_map(mesh_exoid, EX_ELEM_MAP, Data(iElem_Map)) < 0) {
820 fmt::print(stderr, "[{}]: ERROR, unable to output element map\n", __func__);
821 ex_close(mesh_exoid);
822 exit(1);
823 }
824
825 /* If non-nullptr, output the global element id map which preserves
826 the global element ids in the original mesh */
827 if (!globals.Proc_Global_Elem_Id_Map[iproc].empty()) {
828 bytes_out +=
829 globals.Num_Internal_Elems[iproc] * globals.Num_Border_Elems[iproc] * sizeof(INT);
830 if (ex_put_map_param(mesh_exoid, 0, 1) < 0) {
831 fmt::print(stderr, "[{}]: ERROR, unable to define global map parameters!\n", __func__);
832 ex_close(mesh_exoid);
833 exit(1);
834 }
835
836 if (ex_put_num_map(mesh_exoid, EX_ELEM_MAP, 1, Data(globals.Proc_Global_Elem_Id_Map[iproc])) <
837 0) {
838 fmt::print(stderr, "[{}]: ERROR, unable to output global id map!\n", __func__);
839 ex_close(mesh_exoid);
840 exit(1);
841 }
842 if (ex_put_name(mesh_exoid, EX_ELEM_MAP, 1, "original_global_id_map") < 0) {
843 fmt::print(stderr, "[{}]: ERROR, unable to define global map name!\n", __func__);
844 ex_close(mesh_exoid);
845 exit(1);
846 }
847 }
848
849 /* Also output the Nemesis element map */
850 if (ex_put_processor_elem_maps(
851 mesh_exoid, Data(globals.Elem_Map[iproc]),
852 (Data(globals.Elem_Map[iproc]) + globals.Num_Internal_Elems[iproc]), proc_for) < 0) {
853 fmt::print(stderr, "[{}]: ERROR, unable to output nemesis element map!\n", __func__);
854 ex_close(mesh_exoid);
855 exit(1);
856 }
857
858 PIO_Time_Array[12] = (second() - tt1);
859 total_out_time += PIO_Time_Array[12];
860 iElem_Map.clear();
861
862 for (INT i1 = 0; i1 < globals.Num_Elem_Blk; i1++) {
863
864 ex_entity_id iglobal_blk = Elem_Blk_Ids[i1];
865
866 /* Find the local element block index */
867 INT ilocal;
868 for (ilocal = 0; ilocal < globals.Num_Elem_Blk; ilocal++) {
869 if (globals.Proc_Elem_Blk_Ids[iproc][ilocal] == iglobal_blk) {
870 break;
871 }
872 }
873
874 /* Error check */
875 if (ilocal >= globals.Num_Elem_Blk) {
876 fmt::print(stderr, "[{}]: Error finding local element block ID\n", __func__);
877 exit(1);
878 }
879
880 /* If it's a non-null block output all information */
881 if (ilocal < globals.Proc_Num_Elem_Blk[iproc]) {
882 /* Find the first index into the connectivity for this block */
883 size_t iIndex0 = 0;
884 for (INT i2 = 0; i2 < ilocal; i2++) {
885 iIndex0 +=
886 globals.Proc_Num_Elem_In_Blk[iproc][i2] * globals.Proc_Nodes_Per_Elem[iproc][i2];
887 }
888
889 PIO_Time_Array[13] += (second() - tt1);
890
891 size_t tmp_cnt = globals.Proc_Num_Elem_In_Blk[iproc][ilocal] *
892 globals.Proc_Nodes_Per_Elem[iproc][ilocal];
893
894 /* Generate the connectivity array for local node numbering */
895 {
896 std::vector<INT> proc_local_conn(tmp_cnt);
897
898 reverse_map(&globals.Proc_Elem_Connect[iproc][iIndex0], 1, tmp_cnt,
899 Data(globals.GNodes[iproc]), Data(loc_index), Data(proc_local_conn));
900
901 bytes_out += globals.Proc_Nodes_Per_Elem[iproc][ilocal] *
902 globals.Proc_Num_Elem_In_Blk[iproc][ilocal] * sizeof(INT);
903 tt1 = second();
904
905 if (Debug_Flag >= 4) {
906 fmt::print("Putting element_connectivity info in file id: {}\n", mesh_exoid);
907 }
908 if (ex_put_conn(mesh_exoid, EX_ELEM_BLOCK, globals.Proc_Elem_Blk_Ids[iproc][ilocal],
909 Data(proc_local_conn), nullptr, nullptr) < 0) {
910 fmt::print(stderr, "[{}]: ERROR, unable to output connectivity\n", __func__);
911 ex_close(mesh_exoid);
912 exit(1);
913 }
914
915 PIO_Time_Array[14] += (second() - tt1);
916 }
917
918 if (globals.Proc_Num_Attr[iproc][ilocal] > 0) {
919
920 /* Find the first index into the attribute list for this block */
921 size_t iIndex1 = 0;
922 for (INT i2 = 0; i2 < ilocal; i2++) {
923 iIndex1 += globals.Proc_Num_Attr[iproc][i2] * globals.Proc_Num_Elem_In_Blk[iproc][i2];
924 }
925
926 bytes_out += globals.Proc_Num_Elem_In_Blk[iproc][ilocal] * io_ws;
927 tt1 = second();
928
929 T *ptr = &(globals.Proc_Elem_Attr[iproc][iIndex1]);
930
931 if (ex_put_attr(mesh_exoid, EX_ELEM_BLOCK, globals.Proc_Elem_Blk_Ids[iproc][ilocal],
932 ptr) < 0) {
933 fmt::print(stderr, "[{}]: ERROR, unable to output element attributes\n", __func__);
934 exit(1);
935 }
936
937 PIO_Time_Array[15] += (second() - tt1);
938 total_out_time += PIO_Time_Array[15];
939 }
940
941 } /* End "if(ilocal < globals.Num_Elem_Blk[iproc])" */
942
943 } /* End "for(i1=0; i1 < globals.Num_Elem_Block; i1++)" */
944 }
945 total_out_time += (PIO_Time_Array[13] + PIO_Time_Array[14] + PIO_Time_Array[15]);
946
947 /*
948 * Write out the node-set information. Note that the value of the
949 * node-set distribution factor is not currently used so only a
950 * dummy set is output for this value.
951 */
952 INT iMaxLen = 0;
953 for (INT i1 = 0; i1 < globals.Proc_Num_Node_Sets[iproc]; i1++) {
954 iMaxLen = PEX_MAX(globals.Proc_NS_Count[iproc][i1], iMaxLen);
955 }
956
957 /* Renumber Node set node lists to use local node numbers */
958 std::vector<INT> proc_local_ns;
959 if (globals.Proc_Num_Node_Sets[iproc] > 0) {
960 proc_local_ns.resize(globals.Proc_NS_List_Length[iproc]);
961
962 reverse_map(&globals.Proc_NS_List[iproc][0], 1, globals.Proc_NS_List_Length[iproc],
963 Data(globals.GNodes[iproc]), Data(loc_index), Data(proc_local_ns));
964 }
965
966 loc_index.clear();
967
968 PIO_Time_Array[16] = 0.0;
969 PIO_Time_Array[17] = 0.0;
970
971 /* Fill in the information for the nullptr node sets */
972 size_t cnt = 0;
973 for (INT i1 = 0; i1 < globals.Num_Node_Set; i1++) {
974 bool ifound = false;
975 for (INT i2 = 0; i2 < globals.Proc_Num_Node_Sets[iproc]; i2++) {
976 if (globals.Proc_NS_Ids[iproc][i2] == Node_Set_Ids[i1]) {
977 ifound = true;
978 break;
979 }
980 }
981
982 if (!ifound) {
983 globals.Proc_NS_Ids[iproc][globals.Proc_Num_Node_Sets[iproc] + cnt] = Node_Set_Ids[i1];
984 globals.Proc_NS_Count[iproc][globals.Proc_Num_Node_Sets[iproc] + cnt] = 0;
985 globals.Proc_NS_DF_Count[iproc][globals.Proc_Num_Node_Sets[iproc] + cnt] = 0;
986 cnt++;
987 }
988 }
989
990 tt1 = second();
991 if (globals.Num_Node_Set > 0) {
992 size_t dcount = 0;
993 size_t ncount = 0;
994 for (INT i1 = 0; i1 < globals.Num_Node_Set; i1++) {
995 dcount += globals.Proc_NS_DF_Count[iproc][i1];
996 ncount += globals.Proc_NS_Count[iproc][i1];
997 }
998
999 std::vector<INT> conc_ids(globals.Num_Node_Set);
1000 std::vector<INT> conc_nodes(globals.Num_Node_Set);
1001 std::vector<INT> conc_df(globals.Num_Node_Set);
1002 std::vector<INT> conc_nind(globals.Num_Node_Set);
1003 std::vector<INT> conc_dind(globals.Num_Node_Set);
1004 std::vector<INT> conc_nlist(ncount);
1005 std::vector<T> conc_sdf(dcount);
1006
1007 ncount = 0;
1008 dcount = 0;
1009 for (INT i1 = 0; i1 < globals.Num_Node_Set; i1++) {
1010
1011 /* Find the local ID */
1012 INT i2 = 0;
1013 for (i2 = 0; i2 < globals.Num_Node_Set; i2++) {
1014 if (globals.Proc_NS_Ids[iproc][i2] == Node_Set_Ids[i1]) {
1015 break;
1016 }
1017 }
1018
1019 conc_ids[i1] = globals.Proc_NS_Ids[iproc][i2];
1020 conc_nodes[i1] = globals.Proc_NS_Count[iproc][i2];
1021 conc_df[i1] = globals.Proc_NS_DF_Count[iproc][i2];
1022
1023 conc_nind[i1] = ncount;
1024 for (INT i3 = 0; i3 < globals.Proc_NS_Count[iproc][i2]; i3++) {
1025 conc_nlist[ncount++] = proc_local_ns[globals.Proc_NS_Pointers[iproc][i2] + i3];
1026 }
1027
1028 conc_dind[i1] = dcount;
1029 for (INT i3 = 0; i3 < globals.Proc_NS_DF_Count[iproc][i2]; i3++) {
1030 conc_sdf[dcount++] =
1031 globals.Proc_NS_Dist_Fact[iproc][globals.Proc_NS_Pointers[iproc][i2] + i3];
1032 }
1033 }
1034
1035 ex_set_specs set_specs{};
1036 set_specs.sets_ids = Data(conc_ids);
1037 set_specs.num_entries_per_set = Data(conc_nodes);
1038 set_specs.num_dist_per_set = Data(conc_df);
1039 set_specs.sets_entry_index = Data(conc_nind);
1040 set_specs.sets_dist_index = Data(conc_dind);
1041 set_specs.sets_entry_list = Data(conc_nlist);
1042 set_specs.sets_extra_list = nullptr;
1043 set_specs.sets_dist_fact = Data(conc_sdf);
1044 ex_put_concat_sets(mesh_exoid, EX_NODE_SET, &set_specs);
1045 }
1046 total_out_time += second() - tt1;
1047
1048 /* Free local number array */
1049 if (globals.Proc_Num_Node_Sets[iproc] > 0) {
1050 proc_local_ns.clear();
1051 }
1052
1053 /* Renumber element SS to use local element numbers */
1054 std::vector<INT> proc_local_ss;
1055
1056 if (globals.Proc_Num_Side_Sets[iproc] > 0) {
1057 proc_local_ss.resize(globals.Proc_SS_Elem_List_Length[iproc]);
1058 reverse_map(&globals.Proc_SS_Elem_List[iproc][0], 0, globals.Proc_SS_Elem_List_Length[iproc],
1059 Data(globals.GElems[iproc]), (INT *)nullptr, Data(proc_local_ss));
1060 }
1061
1062 PIO_Time_Array[18] = 0.0;
1063 PIO_Time_Array[19] = 0.0;
1064
1065 /* Set up the null side sets */
1066 cnt = 0;
1067 for (INT i1 = 0; i1 < globals.Num_Side_Set; i1++) {
1068 bool ifound = false;
1069 for (INT i2 = 0; i2 < globals.Proc_Num_Side_Sets[iproc]; i2++) {
1070 if (globals.Proc_SS_Ids[iproc][i2] == Side_Set_Ids[i1]) {
1071 ifound = true;
1072 break;
1073 }
1074 }
1075
1076 if (!ifound) {
1077 globals.Proc_SS_Ids[iproc][globals.Proc_Num_Side_Sets[iproc] + cnt] = Side_Set_Ids[i1];
1078 globals.Proc_SS_Elem_Count[iproc][globals.Proc_Num_Side_Sets[iproc] + cnt] = 0;
1079 globals.Proc_SS_DF_Count[iproc][globals.Proc_Num_Side_Sets[iproc] + cnt] = 0;
1080 cnt++;
1081 }
1082 }
1083
1084 /* Output concatenated sidesets. For each processor need:
1085 * side_set ids (size globals.Num_Side_Set)
1086 * num_side_per_set (size globals.Num_Side_Set)
1087 * num_dist_per_set (size globals.Num_Side_Set)
1088 * side_sets_elem_index (size globals.Num_Side_Set)
1089 * side_sets_dist_index (size globals.Num_Side_Set)
1090 * side_sets_elem_list
1091 * side_sets_side_list
1092 * side_sets_dist_fact
1093 */
1094
1095 tt1 = second();
1096 if (globals.Num_Side_Set > 0) {
1097 size_t df_count = 0;
1098 size_t el_count = 0;
1099 for (INT i1 = 0; i1 < globals.Num_Side_Set; i1++) {
1100 df_count += globals.Proc_SS_DF_Count[iproc][i1];
1101 el_count += globals.Proc_SS_Elem_Count[iproc][i1];
1102 }
1103
1104 std::vector<INT> conc_ids(globals.Num_Side_Set);
1105 std::vector<INT> conc_sides(globals.Num_Side_Set);
1106 std::vector<INT> conc_dist(globals.Num_Side_Set);
1107 std::vector<INT> conc_eind(globals.Num_Side_Set);
1108 std::vector<INT> conc_dind(globals.Num_Side_Set);
1109 std::vector<INT> conc_elist(el_count);
1110 std::vector<INT> conc_slist(el_count);
1111 std::vector<T> conc_sdflist(df_count);
1112
1113 /* Fill in the arrays ... */
1114 df_count = 0;
1115 el_count = 0;
1116 for (INT i1 = 0; i1 < globals.Num_Side_Set; i1++) {
1117
1118 /* Find the local ID of this side set */
1119 INT i2 = 0;
1120 for (i2 = 0; i2 < globals.Num_Side_Set; i2++) {
1121 if (globals.Proc_SS_Ids[iproc][i2] == Side_Set_Ids[i1]) {
1122 break;
1123 }
1124 }
1125
1126 conc_ids[i1] = globals.Proc_SS_Ids[iproc][i2];
1127 conc_sides[i1] = globals.Proc_SS_Elem_Count[iproc][i2];
1128 conc_dist[i1] = globals.Proc_SS_DF_Count[iproc][i2];
1129
1130 conc_eind[i1] = el_count;
1131 for (INT i3 = 0; i3 < globals.Proc_SS_Elem_Count[iproc][i2]; i3++) {
1132 conc_elist[el_count] = proc_local_ss[globals.Proc_SS_Elem_Pointers[iproc][i2] + i3];
1133 conc_slist[el_count] =
1134 globals.Proc_SS_Side_List[iproc][globals.Proc_SS_Elem_Pointers[iproc][i2] + i3];
1135 el_count++;
1136 }
1137
1138 conc_dind[i1] = df_count;
1139 for (INT i3 = 0; i3 < globals.Proc_SS_DF_Count[iproc][i2]; i3++) {
1140 conc_sdflist[df_count++] =
1141 globals.Proc_SS_Dist_Fact[iproc][globals.Proc_SS_DF_Pointers[iproc][i2] + i3];
1142 }
1143 }
1144
1145 ex_set_specs set_specs{};
1146 set_specs.sets_ids = Data(conc_ids);
1147 set_specs.num_entries_per_set = Data(conc_sides);
1148 set_specs.num_dist_per_set = Data(conc_dist);
1149 set_specs.sets_entry_index = Data(conc_eind);
1150 set_specs.sets_dist_index = Data(conc_dind);
1151 set_specs.sets_entry_list = Data(conc_elist);
1152 set_specs.sets_extra_list = Data(conc_slist);
1153 set_specs.sets_dist_fact = Data(conc_sdflist);
1154 ex_put_concat_sets(mesh_exoid, EX_SIDE_SET, &set_specs);
1155 }
1156 PIO_Time_Array[19] += (second() - tt1);
1157 total_out_time += (PIO_Time_Array[18] + PIO_Time_Array[19]);
1158
1159 /* Free unneeded memory */
1160 if (globals.Proc_Num_Side_Sets[iproc] > 0) {
1161 proc_local_ss.clear();
1162 }
1163
1164 /*
1165 * Write out the name of the coordinate axes to the parallel ExodusII
1166 * files.
1167 */
1168 bytes_out += globals.Num_Dim * 8 * sizeof(char);
1169 tt1 = second();
1170 if (ex_put_coord_names(mesh_exoid, Coord_Name) < 0) {
1171 fmt::print(stderr, "[{}]: ERROR, could not output coordinate names\n", __func__);
1172 ex_close(mesh_exoid);
1173 exit(1);
1174 }
1175 PIO_Time_Array[20] = (second() - tt1);
1176 total_out_time += PIO_Time_Array[20];
1177
1178 if (Restart_Info.Flag > 0) {
1179
1180 tt1 = second();
1181 bytes_out +=
1182 write_var_param(mesh_exoid, max_name_length, Restart_Info.NVar_Glob, Restart_Info.GV_Name,
1183 Restart_Info.NVar_Node, Restart_Info.NV_Name, Restart_Info.NVar_Elem,
1184 Restart_Info.EV_Name, Data(Restart_Info.GElem_TT), Restart_Info.NVar_Nset,
1185 Restart_Info.NSV_Name, Data(Restart_Info.GNset_TT), Restart_Info.NVar_Sset,
1186 Restart_Info.SSV_Name, Data(Restart_Info.GSset_TT));
1187
1188 PIO_Time_Array[21] = (second() - tt1);
1189 total_out_time += PIO_Time_Array[21];
1190 }
1191
1192 /* Calculate the overall performance */
1193 tt1 = bytes_out;
1194
1195 if (total_out_time == 0) {
1196 tt1 = 0;
1197 }
1198 else {
1199 tt1 = tt1 / total_out_time;
1200 }
1201
1202 tt1 /= 1024.0;
1203
1204 PIO_Time_Array[25] = tt1;
1205
1206 } /* END write_parExo_data() */
1207
1208 template <typename T, typename INT>
1209 int NemSpread<T, INT>::write_var_param(int mesh_exoid, int max_name_length, int num_glob,
1210 char **gv_names, int num_node, char **nv_names, int num_elem,
1211 char **ev_names, int *local_ebtt, int num_nset,
1212 char **ns_names, int *local_nstt, int num_sset,
1213 char **ss_names, int *local_sstt)
1214 {
1215 size_t bytes_out = 0;
1216 int error;
1217
1218 bytes_out += (5 + globals.Num_Elem_Blk * num_elem + globals.Num_Side_Set * num_sset +
1219 globals.Num_Node_Set * num_nset) *
1220 sizeof(INT);
1221 error = ex_put_all_var_param(mesh_exoid, num_glob, num_node, num_elem, local_ebtt, num_nset,
1222 local_nstt, num_sset, local_sstt);
1223 check_exodus_error(error, "ex_put_all_var_param");
1224
1225 if (gv_names != nullptr) {
1226 bytes_out += Restart_Info.NVar_Glob * max_name_length;
1227 error = ex_put_variable_names(mesh_exoid, EX_GLOBAL, num_glob, gv_names);
1228 check_exodus_error(error, "ex_put_var_names");
1229 }
1230 if (nv_names != nullptr) {
1231 bytes_out += num_node * max_name_length;
1232 error = ex_put_variable_names(mesh_exoid, EX_NODAL, num_node, nv_names);
1233 check_exodus_error(error, "ex_put_var_names");
1234 }
1235 if (ev_names != nullptr) {
1236 bytes_out += Restart_Info.NVar_Elem * max_name_length;
1237 error = ex_put_variable_names(mesh_exoid, EX_ELEM_BLOCK, num_elem, ev_names);
1238 check_exodus_error(error, "ex_put_var_names");
1239 }
1240 if (ns_names != nullptr) {
1241 bytes_out += Restart_Info.NVar_Nset * max_name_length;
1242 error = ex_put_variable_names(mesh_exoid, EX_NODE_SET, num_nset, ns_names);
1243 check_exodus_error(error, "ex_put_var_names");
1244 }
1245 if (ss_names != nullptr) {
1246 bytes_out += Restart_Info.NVar_Sset * max_name_length;
1247 error = ex_put_variable_names(mesh_exoid, EX_SIDE_SET, num_sset, ss_names);
1248 check_exodus_error(error, "ex_put_var_names");
1249 }
1250 return (bytes_out);
1251 }
1252
1253 template void NemSpread<double, int>::write_var_timestep(int exoid, int proc, int time_step,
1254 int *eb_ids_global, int *ss_ids_global,
1255 int *ns_ids_global);
1256 template void NemSpread<float, int>::write_var_timestep(int exoid, int proc, int time_step,
1257 int *eb_ids_global, int *ss_ids_global,
1258 int *ns_ids_global);
1259 template void NemSpread<double, int64_t>::write_var_timestep(int exoid, int proc, int time_step,
1260 int64_t *eb_ids_global,
1261 int64_t *ss_ids_global,
1262 int64_t *ns_ids_global);
1263 template void NemSpread<float, int64_t>::write_var_timestep(int exoid, int proc, int time_step,
1264 int64_t *eb_ids_global,
1265 int64_t *ss_ids_global,
1266 int64_t *ns_ids_global);
1267
1268 template <typename T, typename INT>
1269 void NemSpread<T, INT>::write_var_timestep(int exoid, int proc, int time_step, INT *eb_ids_global,
1270 INT *ss_ids_global, INT *ns_ids_global)
1271 {
1272 int error;
1273
1274 /* output the time */
1275 {
1276 T *var_ptr = (T *)&(Restart_Info.Time);
1277 error = ex_put_time(exoid, time_step, var_ptr);
1278 check_exodus_error(error, "ex_put_time");
1279 }
1280
1281 /* start by outputting the global variables */
1282 if (Restart_Info.NVar_Glob > 0) {
1283
1284 T *var_ptr = Data(Restart_Info.Glob_Vals);
1285
1286 error = ex_put_var(exoid, time_step, EX_GLOBAL, 1, 0, Restart_Info.NVar_Glob, var_ptr);
1287
1288 check_exodus_error(error, "ex_put_glob_vars");
1289 }
1290
1291 if (Restart_Info.NVar_Node > 0) {
1292 size_t num_nodes = globals.Num_Internal_Nodes[proc] + globals.Num_Border_Nodes[proc] +
1293 globals.Num_External_Nodes[proc];
1294
1295 for (int var_num = 0; var_num < Restart_Info.NVar_Node; var_num++) {
1296
1297 size_t var_offset = var_num * num_nodes;
1298
1299 T *var_ptr = &(Restart_Info.Node_Vals[proc][var_offset]);
1300
1301 error = ex_put_var(exoid, time_step, EX_NODAL, (var_num + 1), 1, num_nodes, var_ptr);
1302
1303 check_exodus_error(error, "ex_put_var");
1304 }
1305 }
1306
1307 if (Restart_Info.NVar_Elem > 0) {
1308
1309 size_t num_elem = globals.Num_Internal_Elems[proc] + globals.Num_Border_Elems[proc];
1310
1311 for (int var_num = 0; var_num < Restart_Info.NVar_Elem; var_num++) {
1312 int eb_num_g = 0;
1313
1314 size_t var_offset = var_num * num_elem;
1315 T *var_ptr = &(Restart_Info.Elem_Vals[proc][var_offset]);
1316
1317 for (int eb_num = 0; eb_num < globals.Proc_Num_Elem_Blk[proc]; eb_num++) {
1318
1319 /* now I have to find the appropriate entry in the truth table */
1320 /* can always assume this eb num is greater than the last one */
1321 for (int cnt1 = eb_num_g; cnt1 < globals.Num_Elem_Blk; cnt1++) {
1322 if (globals.Proc_Elem_Blk_Ids[proc][eb_num] == eb_ids_global[cnt1]) {
1323 eb_num_g = cnt1;
1324 break;
1325 }
1326 }
1327
1328 if (Restart_Info.GElem_TT[eb_num_g * Restart_Info.NVar_Elem + var_num]) {
1329
1330 error = ex_put_var(exoid, time_step, EX_ELEM_BLOCK, (var_num + 1),
1331 globals.Proc_Elem_Blk_Ids[proc][eb_num],
1332 globals.Proc_Num_Elem_In_Blk[proc][eb_num], var_ptr);
1333
1334 check_exodus_error(error, "ex_put_elem_var");
1335 }
1336 /* and now move the variable pointer */
1337
1338 /* Note that the offsetting here must match the 'var_offset'
1339 * treatment in ps_restart.c function read_elem_vars.
1340 * Currently, the offset is applied even if the variable does
1341 * not exist on a particular block.
1342 */
1343 var_ptr += globals.Proc_Num_Elem_In_Blk[proc][eb_num];
1344 }
1345 }
1346 }
1347
1348 if (Restart_Info.NVar_Sset > 0) {
1349 int ss_num_g = 0;
1350 size_t num_elem = globals.Proc_SS_Elem_List_Length[proc];
1351 for (int var_num = 0; var_num < Restart_Info.NVar_Sset; var_num++) {
1352
1353 size_t var_offset = var_num * num_elem;
1354 T *var_ptr = &(Restart_Info.Sset_Vals[proc][var_offset]);
1355
1356 for (int ss_num = 0; ss_num < globals.Proc_Num_Side_Sets[proc]; ss_num++) {
1357
1358 /* now I have to find the appropriate entry in the truth table */
1359 for (int cnt1 = 0; cnt1 < globals.Num_Side_Set; cnt1++) {
1360 if (globals.Proc_SS_Ids[proc][ss_num] == ss_ids_global[cnt1]) {
1361 ss_num_g = cnt1;
1362 break;
1363 }
1364 }
1365 assert(globals.Proc_SS_Ids[proc][ss_num] == ss_ids_global[ss_num_g]);
1366
1367 if (Restart_Info.GSset_TT[ss_num_g * Restart_Info.NVar_Sset + var_num]) {
1368
1369 error = ex_put_var(exoid, time_step, EX_SIDE_SET, (var_num + 1),
1370 globals.Proc_SS_Ids[proc][ss_num],
1371 globals.Proc_SS_Elem_Count[proc][ss_num], var_ptr);
1372
1373 check_exodus_error(error, "ex_put_sset_var");
1374 }
1375 /* and now move the variable pointer */
1376
1377 /* Note that the offsetting here must match the 'var_offset'
1378 * treatment in ps_restart.c function read_elem_vars.
1379 * Currently, the offset is applied even if the variable does
1380 * not exist on a particular block.
1381 */
1382 var_ptr += globals.Proc_SS_Elem_Count[proc][ss_num];
1383 }
1384 }
1385 }
1386
1387 if (Restart_Info.NVar_Nset > 0) {
1388 int ns_num_g = 0;
1389 size_t num_elem = globals.Proc_NS_List_Length[proc];
1390 for (int var_num = 0; var_num < Restart_Info.NVar_Nset; var_num++) {
1391
1392 size_t var_offset = var_num * num_elem;
1393 T *var_ptr = &(Restart_Info.Nset_Vals[proc][var_offset]);
1394
1395 for (int ns_num = 0; ns_num < globals.Proc_Num_Node_Sets[proc]; ns_num++) {
1396
1397 /* now I have to find the appropriate entry in the truth table */
1398 for (int cnt1 = 0; cnt1 < globals.Num_Node_Set; cnt1++) {
1399 if (globals.Proc_NS_Ids[proc][ns_num] == ns_ids_global[cnt1]) {
1400 ns_num_g = cnt1;
1401 break;
1402 }
1403 }
1404 assert(globals.Proc_NS_Ids[proc][ns_num] == ns_ids_global[ns_num_g]);
1405
1406 if (Restart_Info.GNset_TT[ns_num_g * Restart_Info.NVar_Nset + var_num]) {
1407
1408 error = ex_put_var(exoid, time_step, EX_NODE_SET, (var_num + 1),
1409 globals.Proc_NS_Ids[proc][ns_num], globals.Proc_NS_Count[proc][ns_num],
1410 var_ptr);
1411
1412 check_exodus_error(error, "ex_put_nset_var");
1413 }
1414 /* and now move the variable pointer */
1415
1416 /* Note that the offsetting here must match the 'var_offset'
1417 * treatment in ps_restart.c function read_elem_vars.
1418 * Currently, the offset is applied even if the variable does
1419 * not exist on a particular block.
1420 */
1421 var_ptr += globals.Proc_NS_Count[proc][ns_num];
1422 }
1423 }
1424 }
1425 }
1426
1427 namespace {
1428 template <typename INT>
1429 void reverse_map(const std::vector<INT> &global, int p01, size_t gsize,
1430 const std::vector<INT> &glmap, INT *index, std::vector<INT> &mapout)
1431 {
1432 /*
1433 * The 'global' array is an array of node or element numbers
1434 * in the global id space. It needs to be converted to local
1435 * numbers via the 'glmap' array. The glmap array is sorted
1436 * by the 'index' array. The map from global to local is
1437 * glmap[local_id] = global_id
1438 *
1439 * The 'p01' is either 0 or 1 and is an offset to the values
1440 * in 'global'
1441 */
1442
1443 /*
1444 * The algorithm used is to sort the 'global' array via the
1445 * 'tmp_index' array (global[tmp_index[0..gsize]] is sorted)
1446 * Then, progress through the 'global' array in sorted order
1447 * and find the location in 'glmap'. Note that since both are
1448 * sorted, it should be easy to progress sequentially through
1449 * both arrays.
1450 */
1451
1452 std::vector<INT> tmp_index(gsize);
1453
1454 /* Initialize index array */
1455 for (size_t i2 = 0; i2 < gsize; i2++) {
1456 tmp_index[i2] = (INT)i2;
1457 }
1458
1459 /* Sort the 'global' array via the index array 'tmp_index' */
1460 gds_iqsort(Data(global), Data(tmp_index), gsize);
1461
1462 size_t i3 = 0;
1463 if (index != nullptr) {
1464 for (size_t i2 = 0; i2 < gsize; i2++) {
1465 INT gval = global[tmp_index[i2]] + p01;
1466
1467 while (glmap[index[i3]] < gval) {
1468 i3++;
1469 }
1470
1471 assert(glmap[index[i3]] == gval);
1472
1473 mapout[tmp_index[i2]] = index[i3] + 1;
1474 }
1475 }
1476 else {
1477 for (size_t i2 = 0; i2 < gsize; i2++) {
1478 INT gval = global[tmp_index[i2]] + p01;
1479
1480 while (glmap[i3] < gval) {
1481 i3++;
1482 }
1483
1484 assert(glmap[i3] == gval);
1485
1486 mapout[tmp_index[i2]] = i3 + 1;
1487 }
1488 }
1489 }
1490
1491 template <typename INT>
1492 void reverse_map(INT *global, int p01, size_t gsize, INT *glmap, INT *index, INT *mapout)
1493 {
1494 /*
1495 * The 'global' array is an array of node or element numbers
1496 * in the global id space. It needs to be converted to local
1497 * numbers via the 'glmap' array. The glmap array is sorted
1498 * by the 'index' array. The map from global to local is
1499 * glmap[local_id] = global_id
1500 *
1501 * The 'p01' is either 0 or 1 and is an offset to the values
1502 * in 'global'
1503 */
1504
1505 /*
1506 * The algorithm used is to sort the 'global' array via the
1507 * 'tmp_index' array (global[tmp_index[0..gsize]] is sorted)
1508 * Then, progress through the 'global' array in sorted order
1509 * and find the location in 'glmap'. Note that since both are
1510 * sorted, it should be easy to progress sequentially through
1511 * both arrays.
1512 */
1513
1514 std::vector<INT> tmp_index(gsize);
1515 std::iota(tmp_index.begin(), tmp_index.end(), 0);
1516
1517 /* Sort the 'global' array via the index array 'tmp_index' */
1518 gds_iqsort(global, Data(tmp_index), gsize);
1519
1520 #if 0
1521 for (size_t i2 = 0; i2 < gsize; i2++) {
1522 INT gval = global[tmp_index[i2]] + p01;
1523 fmt::print(stderr, "GLOBAL: {} {} {}\n", i2, tmp_index[i2], gval);
1524 }
1525 #endif
1526
1527 size_t i3 = 0;
1528 if (index != nullptr) {
1529 for (size_t i2 = 0; i2 < gsize; i2++) {
1530 INT gval = global[tmp_index[i2]] + p01;
1531
1532 while (glmap[index[i3]] < gval) {
1533 i3++;
1534 }
1535
1536 assert(glmap[index[i3]] == gval);
1537
1538 mapout[tmp_index[i2]] = index[i3] + 1;
1539 }
1540 }
1541 else {
1542 for (size_t i2 = 0; i2 < gsize; i2++) {
1543 INT gval = global[tmp_index[i2]] + p01;
1544
1545 while (glmap[i3] < gval) {
1546 i3++;
1547 }
1548
1549 assert(glmap[i3] == gval);
1550
1551 mapout[tmp_index[i2]] = i3 + 1;
1552 }
1553 }
1554 }
1555 } // namespace