Debian Codegraph @

packages/seacas/applications/nem_spread/nem_spread.C

raw
1 /*
2 * Copyright(C) 1999-2021, 2023, 2025 National Technology & Engineering Solutions
3 * of Sandia, LLC (NTESS). Under the terms of Contract DE-NA0003525 with
4 * NTESS, the U.S. Government retains certain rights in this software.
5 *
6 * See packages/seacas/LICENSE for details
7 */
8 /*--------------------------------------------------------------------------*/
9 /* This program takes an Exodus II FEM database file and a parallel */
10 /* load-balance file and creates a set of parallel Exodus II FEM files for */
11 /* use by SALSA. */
12 /*--------------------------------------------------------------------------*/
13
14 #include "add_to_log.h" // for add_to_log
15 #include "exodusII.h" // for ex_opts, ex_int64_status, etc
16 #include "fmt/ostream.h"
17 #include "nem_spread.h" // for NemSpread, second, etc
18 #include "ps_pario.h" // for PIO_Info
19 #include "ps_pario_const.h" // for Parallel_IO
20 #include "rf_allo.h" // for safe_free
21 #include "rf_io.h" // for ExoFile, Debug_Flag, etc
22 #include <cstdint> // for int64_t
23 #include <cstdio> // for stderr, etc
24 #include <cstdlib> // for exit
25 #include <cstring>
26 #include <unistd.h> // for getopt, optarg, optind
27
28 #if defined(WIN32) || defined(__WIN32__) || defined(_WIN32) || defined(_MSC_VER) || \
29 defined(__MINGW32__) || defined(_WIN64) || defined(__MINGW64__)
30 #include "XGetopt.h"
31 #else
32 #include "getopt.h" // for getopt
33 #endif
34
35 namespace {
36 int check_change_sets(int exoid, int &selected_change_set)
37 {
38 // First, see if the file contains change sets...
39 int num_change_sets = ex_inquire_int(exoid, EX_INQ_NUM_CHILD_GROUPS);
40
41 if (num_change_sets == 0) {
42 if (selected_change_set > 0) {
43 fmt::print(stderr,
44 "ERROR: Reading from change set {} was specified, but the file contains no "
45 "change sets.\n",
46 selected_change_set);
47 return 0;
48 }
49 return exoid;
50 }
51
52 // File contains change sets...
53 if (selected_change_set == 0) {
54 fmt::print(
55 stderr,
56 "\nWARNING: Exodus database contains {} change sets.\n Setting to read from "
57 "first change set since `-change_set #` option not specified.\n\n",
58 num_change_sets);
59 selected_change_set = 1;
60 return exoid + selected_change_set;
61 }
62
63 if (selected_change_set > num_change_sets) {
64 fmt::print(stderr,
65 "ERROR: Change set {} was specified for reading, but mesh only contains {} change "
66 "sets.\n",
67 selected_change_set, num_change_sets);
68 return 0;
69 }
70 // Contains change sets and selected change set is in range...
71 fmt::print(stderr, "NOTE: Mesh data will be read from change set {} of {}\n",
72 selected_change_set, num_change_sets);
73 return exoid + selected_change_set;
74 }
75 } // namespace
76 extern int read_mesh_file_name(const char *filename);
77
78 template <typename T, typename INT>
79 int read_pexoII_info(NemSpread<T, INT> &spreader, const char *filename);
80
81 template <typename T, typename INT>
82 int nem_spread(NemSpread<T, INT> &spreader, const char *salsa_cmd_file, int subcycles, int cycle);
83
84 int main(int argc, char *argv[])
85 {
86 const char *salsa_cmd_file;
87 int c;
88
89 double g_start_t = second();
90 bool force_64_bit = false;
91 int start_proc = 0;
92 int num_proc = 0;
93 int subcycles = 0;
94 int cycle = -1;
95 int selected_change_set = 0;
96
97 while ((c = getopt(argc, argv, "64Vhp:r:s:n:S:c:C:")) != -1) {
98 switch (c) {
99 case 'h':
100 fmt::print(stderr, " usage:\n");
101 fmt::print(stderr,
102 "\tnem_spread [-s <start_proc>] [-n <num_proc>] [-S <subcycles> -c <cycle>] "
103 "[-C change_set_#] [command_file]\n");
104 fmt::print(stderr, "\t\tDecompose for processors <start_proc> to <start_proc>+<num_proc>\n");
105 fmt::print(stderr, "\t\tDecompose for cycle <cycle> of <subcycle> groups\n");
106 fmt::print(stderr, "\t\tRead from change_set `#` (1-based) if specified.\n");
107 fmt::print(stderr, "\tnem_spread [-V] [-h] (show version or usage info)\n");
108 fmt::print(stderr, "\tnem_spread [command file] [<-p Proc> <-r raid #>]\n");
109 exit(1);
110 break;
111 case 'V':
112 fmt::print("{} version {}\n", UTIL_NAME, VER_STR);
113 exit(0);
114 break;
115 case 'p': /* Which proc to use? Also for compatibility */ break;
116 case 'r': /* raid number. Seems to be unused; left around for compatibility */ break;
117 case 's': /* Start with processor <x> */ sscanf(optarg, "%d", &start_proc); break;
118 case 'n': /* Number of processors to output files for */ sscanf(optarg, "%d", &num_proc); break;
119 case 'C': /* change_set index <x> */ sscanf(optarg, "%d", &selected_change_set); break;
120 case '6':
121 case '4':
122 force_64_bit = true; /* Force storing output mesh using 64bit integers */
123 break;
124 case 'S': /* Number of subcycles to use (see below) */ sscanf(optarg, "%d", &subcycles); break;
125 case 'c': /* Which cycle to spread (see below) */ sscanf(optarg, "%d", &cycle); break;
126 }
127 }
128
129 if (optind >= argc) {
130 salsa_cmd_file = "nem_spread.inp";
131 }
132 else {
133 salsa_cmd_file = argv[optind];
134 }
135
136 fmt::print("{} version {}\n", UTIL_NAME, VER_STR);
137
138 /* initialize some variables */
139 ExoFile[0] = '\0';
140 Exo_Res_File[0] = '\0';
141 Debug_Flag = -1;
142
143 Num_Nod_Var = -1;
144 Num_Elem_Var = -1;
145 Num_Glob_Var = -1;
146 Num_Nset_Var = -1;
147 Num_Sset_Var = -1;
148
149 PIO_Info.Dsk_List_Cnt = -1;
150 PIO_Info.Num_Dsk_Ctrlrs = -1;
151 PIO_Info.PDsk_Add_Fact = -1;
152 PIO_Info.Zeros = -1;
153 PIO_Info.NoSubdirectory = 0;
154 PIO_Info.Par_Dsk_Root[0] = '\0';
155 PIO_Info.Par_Dsk_SubDirec[0] = '\0';
156 PIO_Info.Staged_Writes = true;
157
158 static char yo[] = "nem_spread";
159 // Read the ASCII input file and get the name of the mesh file
160 // so we can determine the floating point and integer word sizes
161 // needed to instantiate the templates...
162 if (read_mesh_file_name(salsa_cmd_file) < 0) {
163 fmt::print(stderr,
164 "{} ERROR: Could not read in the the I/O command file"
165 " \"{}\"!\n",
166 yo, salsa_cmd_file);
167 exit(1);
168 }
169
170 // Open the mesh file and determine word sizes...
171 int io_ws = 0;
172 int cpu_ws = sizeof(float);
173 float version;
174
175 int exoid = ex_open(ExoFile.c_str(), EX_READ, &cpu_ws, &io_ws, &version);
176 if (exoid <= 0) {
177 fmt::print(stderr, "{} ERROR: Could not open the mesh file '{}'!\n", yo, ExoFile);
178 exit(1);
179 }
180
181 // Determine whether there are any change sets in file.
182 // If there are, check whether user specified a specific
183 // change set index and set to that one (if valid), or
184 // if not specified, set to the first.
185 exoid = check_change_sets(exoid, selected_change_set);
186 if (exoid == 0) {
187 exit(1);
188 }
189
190 // See if any 64-bit integers stored on database...
191 int int64api = 0;
192 int int64db = ex_int64_status(exoid) & EX_ALL_INT64_DB;
193 if (int64db != 0) {
194 int64api = EX_ALL_INT64_API;
195 force_64_bit = true;
196 }
197
198 int status;
199 if (io_ws == 4) {
200 if (int64api != 0) {
201 NemSpread<float, int64_t> spreader;
202 spreader.selected_change_set = selected_change_set;
203 spreader.io_ws = io_ws;
204 spreader.int64db = int64db;
205 spreader.int64api = int64api;
206 spreader.force64db = force_64_bit;
207 spreader.Proc_Info[4] = start_proc;
208 spreader.Proc_Info[5] = num_proc;
209 status = nem_spread(spreader, salsa_cmd_file, subcycles, cycle);
210 }
211 else {
212 NemSpread<float, int> spreader;
213 spreader.selected_change_set = selected_change_set;
214 spreader.io_ws = io_ws;
215 spreader.int64db = int64db;
216 spreader.int64api = int64api;
217 spreader.force64db = force_64_bit;
218 spreader.Proc_Info[4] = start_proc;
219 spreader.Proc_Info[5] = num_proc;
220 status = nem_spread(spreader, salsa_cmd_file, subcycles, cycle);
221 }
222 }
223 else {
224 if (int64api != 0) {
225 NemSpread<double, int64_t> spreader;
226 spreader.selected_change_set = selected_change_set;
227 spreader.io_ws = io_ws;
228 spreader.int64db = int64db;
229 spreader.int64api = int64api;
230 spreader.force64db = force_64_bit;
231 spreader.Proc_Info[4] = start_proc;
232 spreader.Proc_Info[5] = num_proc;
233 status = nem_spread(spreader, salsa_cmd_file, subcycles, cycle);
234 }
235 else {
236 NemSpread<double, int> spreader;
237 spreader.selected_change_set = selected_change_set;
238 spreader.io_ws = io_ws;
239 spreader.int64db = int64db;
240 spreader.int64api = int64api;
241 spreader.force64db = force_64_bit;
242 spreader.Proc_Info[4] = start_proc;
243 spreader.Proc_Info[5] = num_proc;
244 status = nem_spread(spreader, salsa_cmd_file, subcycles, cycle);
245 }
246 }
247 double g_end_t = second() - g_start_t;
248 fmt::print("The average run time was: {:4f}s\n", g_end_t);
249
250 ex_close(exoid);
251 add_to_log(argv[0], g_end_t);
252 return status;
253 }
254
255 template <typename T, typename INT>
256 int nem_spread(NemSpread<T, INT> &spreader, const char *salsa_cmd_file, int subcycles, int cycle)
257 {
258 static char yo[] = "nem_spread";
259 /* Local declarations. */
260 double start_t;
261 double end_t;
262
263 fmt::print("Using {} byte integers and {} byte floating point values.\n", sizeof(INT), sizeof(T));
264
265 /*
266 * Read in the ASCII input file from the front end.
267 * NOTE: In this case we read only the information needed to create the
268 * parallel exodus files.
269 */
270 fmt::print("Reading the command file, {}\n", salsa_cmd_file);
271 if (read_pexoII_info(spreader, salsa_cmd_file) < 0) {
272 fmt::print(stderr,
273 "{} ERROR: Could not read in the the I/O command file"
274 " \"{}\"!\n",
275 yo, salsa_cmd_file);
276 exit(1);
277 }
278
279 if (!spreader.check_inp()) {
280 fmt::print(stderr, "{} ERROR: Error in user specified parameters.\n", yo);
281 exit(1);
282 }
283
284 /* If debug is on the turn on netCDF/Exodus information as well */
285 if (Debug_Flag > 0) {
286 ex_opts(EX_VERBOSE | EX_DEBUG);
287 }
288 else {
289 ex_opts(EX_VERBOSE);
290 }
291
292 /*
293 * Read initial information from the mesh file.
294 * - This provides error checking against the load balance file
295 * - Broadcast the information
296 */
297 spreader.read_mesh_param();
298
299 /*
300 * Process the load balance information
301 * - Read info from the lb file with Proc 0
302 * - Distribute the information to all processors
303 */
304 start_t = second();
305 spreader.load_lb_info();
306 end_t = second() - start_t;
307 fmt::print("\nLoad load balance information time: {} (sec.)\n\n", end_t);
308
309 /* Process subcycle and cycle if specified. */
310 if ((subcycles > 0 && cycle == -1) || (cycle != -1 && subcycles == 0)) {
311 fmt::print(stderr, "ERROR: Only one of the -subcycle and -cycle options was specified.\n");
312 fmt::print(stderr, " Either both or neither are required.\n");
313 exit(1);
314 }
315
316 if (subcycles > 0) {
317 /* Determine number of processors per subcycle. */
318 int part_count = (spreader.Proc_Info[0] + subcycles - 1) / subcycles;
319 int start_part = part_count * cycle;
320 spreader.Proc_Info[4] = start_part;
321 spreader.Proc_Info[5] = part_count;
322 }
323
324 /*
325 * Verify parameters in case spreading a subset of mesh...
326 */
327 if (spreader.Proc_Info[4] < 0) {
328 spreader.Proc_Info[4] = 0;
329 }
330 if (spreader.Proc_Info[5] <= 0) {
331 spreader.Proc_Info[5] = spreader.Proc_Info[0];
332 }
333
334 if (spreader.Proc_Info[4] + spreader.Proc_Info[5] > spreader.Proc_Info[0]) {
335 spreader.Proc_Info[5] = spreader.Proc_Info[0] - spreader.Proc_Info[4];
336 }
337
338 if (spreader.Proc_Info[4] != 0 || spreader.Proc_Info[5] != spreader.Proc_Info[0]) {
339 fmt::print(
340 "\nSpreading subset of mesh. Starting with processor {} and outputting {} processors.\n",
341 spreader.Proc_Info[4], spreader.Proc_Info[5]);
342 }
343
344 /*
345 * Get any restart parameter information
346 * - Read the parameters from the input ExodusII file
347 */
348 if (spreader.Restart_Info.Flag > 0) {
349 fmt::print("Load exoII restart param info to each proc.\n\n");
350 start_t = second();
351 spreader.read_restart_params();
352 end_t = second() - start_t;
353 fmt::print("Load restart parameters time: {} (sec.)\n\n", end_t);
354 }
355
356 /*
357 * Read the ExodusII mesh file and distribute information
358 * contained in it to all processors
359 * - Each processor only gets the information that it needs
360 * to solve its piece of the mesh
361 */
362 fmt::print("Load exoII mesh info to each proc.\n\n");
363 start_t = second();
364 spreader.load_mesh();
365 end_t = second() - start_t;
366
367 fmt::print("Load mesh time: {} (sec.)\n\n", end_t);
368
369 /*
370 * Get any restart variable data
371 * - Read the restart data from the input ExodusII file, distribute
372 * it to the processors, and write it to the parallel files
373 */
374 if (spreader.Restart_Info.Flag > 0) {
375 fmt::print("Load exoII restart data info to each proc.\n\n");
376 start_t = second();
377 spreader.read_restart_data();
378 end_t = second() - start_t;
379 fmt::print("Load restart data time: {} (sec.)\n\n", end_t);
380 }
381
382 fmt::print("Write of parallel exodus complete\n");
383
384 safe_free(reinterpret_cast<void **>(&(PIO_Info.RDsk_List)));
385
386 for (int i = 0; i < spreader.Proc_Info[0]; i++) {
387 if (spreader.globals.Elem_Type != nullptr) {
388 safe_free((void **)&spreader.globals.Elem_Type[i]);
389 }
390 safe_free((void **)&spreader.globals.Proc_SS_Ids[i]);
391 safe_free((void **)&spreader.globals.Proc_SS_GEMap_List[i]);
392 safe_free((void **)&spreader.globals.Proc_NS_Ids[i]);
393 safe_free((void **)&spreader.globals.Proc_NS_GNMap_List[i]);
394 safe_free((void **)&spreader.globals.Proc_Nodes_Per_Elem[i]);
395 safe_free((void **)&spreader.globals.GElem_Blks[i]);
396 spreader.globals.Proc_Global_Elem_Id_Map[i].clear();
397 spreader.globals.Proc_Global_Node_Id_Map[i].clear();
398 }
399 safe_free((void **)&spreader.globals.Elem_Type);
400 return 0;
401 }