Debian Codegraph @

packages/seacas/applications/slice/SL_Decompose.C

raw
1 // Copyright(C) 1999-2024 National Technology & Engineering Solutions
2 // of Sandia, LLC (NTESS). Under the terms of Contract DE-NA0003525 with
3 // NTESS, the U.S. Government retains certain rights in this software.
4 //
5 // See packages/seacas/LICENSE for details
6
7 #include <SL_Decompose.h>
8 #include <SL_tokenize.h>
9 #include <random>
10
11 #include <Ioss_CodeTypes.h>
12 #include <Ioss_DatabaseIO.h>
13 #include <Ioss_DecompositionUtils.h>
14 #include <Ioss_ElementBlock.h>
15 #include <Ioss_NodeBlock.h>
16 #include <Ioss_ParallelUtils.h>
17 #include <Ioss_Region.h>
18 #include <Ioss_SmartAssert.h>
19 #include <Ioss_Utils.h>
20
21 #include <cassert>
22 #include <fstream>
23
24 #include <exodusII.h>
25 #include <fmt/format.h>
26 #include <fmt/ostream.h>
27 #include <fmt/ranges.h>
28 #include <init/Ionit_Initializer.h>
29
30 #if USE_METIS
31 #include <metis.h>
32 #else
33 using idx_t = int;
34 #endif
35
36 extern int debug_level;
37 extern double seacas_timer();
38 extern void progress(const std::string &output);
39
40 namespace {
41 char **get_name_array(size_t count, int size)
42 {
43 auto *names = new char *[count];
44 for (size_t i = 0; i < count; i++) {
45 names[i] = new char[size + 1];
46 std::memset(names[i], '\0', size + 1);
47 }
48 return names;
49 }
50
51 void delete_name_array(char **names, int count)
52 {
53 for (int i = 0; i < count; i++) {
54 delete[] names[i];
55 }
56 delete[] names;
57 }
58
59 template <typename INT>
60 void create_adjacency_list(const Ioss::Region &region, std::vector<idx_t> &pointer,
61 std::vector<idx_t> &adjacency, INT)
62 {
63 progress(__func__);
64 // Size of pointer list is element count + 1;
65 // Size of adjacency list is sum of nodes-per-element for each element.
66 size_t sum = 0;
67 size_t count = 0;
68 const auto &ebs = region.get_element_blocks();
69 for (const auto &eb : ebs) {
70 size_t element_count = eb->entity_count();
71 size_t element_nodes = eb->topology()->number_nodes();
72 sum += element_count * element_nodes;
73 count += element_count;
74 }
75
76 pointer.reserve(count + 1);
77 adjacency.reserve(sum);
78 fmt::print(stderr, "\tAdjacency Size = {} for {} elements.\n", fmt::group_digits(sum),
79 fmt::group_digits(count));
80
81 // Now, iterate the blocks again, get connectivity and build adjacency structure.
82 std::vector<INT> connectivity;
83 for (const auto &eb : ebs) {
84 eb->get_field_data("connectivity_raw", connectivity);
85 size_t element_count = eb->entity_count();
86 size_t element_nodes = eb->topology()->number_nodes();
87
88 size_t el = 0;
89 for (size_t j = 0; j < element_count; j++) {
90 pointer.push_back(adjacency.size());
91 for (size_t k = 0; k < element_nodes; k++) {
92 INT node = connectivity[el++] - 1;
93 adjacency.push_back(node);
94 }
95 }
96 }
97 pointer.push_back(adjacency.size());
98 assert(pointer.size() == count + 1);
99 assert(adjacency.size() == sum);
100 }
101
102 void exodus_error(int lineno)
103 {
104 std::ostringstream errmsg;
105 fmt::print(
106 errmsg,
107 "Exodus error ({}) {} at line {} in file Slice.C. Please report to [email protected] "
108 "if you need help.",
109 exerrval, ex_strerror(exerrval), lineno);
110
111 ex_err(nullptr, nullptr, EX_PRTLASTMSG);
112 throw std::runtime_error(errmsg.str());
113 }
114
115 bool case_compare(const std::string &s1, const std::string &s2)
116 {
117 return (s1.size() == s2.size()) &&
118 std::equal(s1.begin(), s1.end(), s2.begin(),
119 [](char a, char b) { return std::tolower(a) == std::tolower(b); });
120 }
121
122 #if USE_METIS
123 int get_common_node_count(const Ioss::Region &region)
124 {
125 progress(__func__);
126 // Determine number of nodes that elements must share to be
127 // considered connected. A 8-node hex-only mesh would have 4
128 // A 3D shell mesh should have 2. Basically, use the minimum
129 // number of nodes per side for all element blocks... Omit sphere
130 // elements; ignore bars(?)...
131
132 int common_nodes = 999;
133
134 const auto &ebs = region.get_element_blocks();
135 for (const auto &eb : ebs) {
136 const Ioss::ElementTopology *topology = eb->topology();
137 const Ioss::ElementTopology *boundary = topology->boundary_type(0);
138 if (boundary != nullptr) {
139 common_nodes = std::min(common_nodes, boundary->number_boundaries());
140 }
141 else {
142 // Different topologies on some element faces...
143 size_t nb = topology->number_boundaries();
144 for (size_t bb = 1; bb <= nb; bb++) {
145 boundary = topology->boundary_type(bb);
146 if (boundary != nullptr) {
147 common_nodes = std::min(common_nodes, boundary->number_boundaries());
148 }
149 }
150 }
151 }
152
153 common_nodes = std::max(1, common_nodes);
154 fmt::print(stderr, "Setting common_nodes to {}\n", common_nodes);
155 return common_nodes;
156 }
157
158 template <typename INT>
159 void decompose_metis(const Ioss::Region &region, SystemInterface &interFace,
160 std::vector<int> &elem_to_proc, INT dummy)
161 {
162 size_t element_count = region.get_property("element_count").get_int();
163
164 std::vector<idx_t> pointer;
165 std::vector<idx_t> adjacency;
166
167 double start = seacas_timer();
168 create_adjacency_list(region, pointer, adjacency, dummy);
169 double end = seacas_timer();
170 fmt::print(stderr, "\tCreate Adjacency List = {:.5}\n", end - start);
171
172 // Call Metis to get the partition...
173 {
174 start = seacas_timer();
175 idx_t elem_count = element_count;
176 idx_t common = get_common_node_count(region);
177 idx_t proc_count = interFace.processor_count();
178 idx_t obj_val = 0;
179 std::vector<idx_t> options((METIS_NOPTIONS));
180 METIS_SetDefaultOptions(&options[0]);
181 if (interFace.decomposition_method() == "kway") {
182 options[METIS_OPTION_PTYPE] = METIS_PTYPE_KWAY;
183 }
184 else {
185 options[METIS_OPTION_PTYPE] = METIS_PTYPE_RB;
186 }
187
188 options[METIS_OPTION_OBJTYPE] = METIS_OBJTYPE_CUT;
189 if (interFace.contiguous_decomposition()) {
190 options[METIS_OPTION_CONTIG] = 1;
191 }
192 options[METIS_OPTION_DBGLVL] = 2;
193 options[METIS_OPTION_MINCONN] = 1;
194
195 idx_t node_count = region.get_property("node_count").get_int();
196 std::vector<idx_t> node_partition(node_count);
197 std::vector<idx_t> elem_partition(element_count);
198
199 fmt::print(stderr, "\tCalling METIS Decomposition routine.\n");
200
201 METIS_PartMeshDual(&elem_count, &node_count, &pointer[0], &adjacency[0], nullptr, nullptr,
202 &common, &proc_count, nullptr, &options[0], &obj_val, &elem_partition[0],
203 &node_partition[0]);
204
205 Ioss::Utils::clear(node_partition);
206 elem_to_proc.reserve(element_count);
207 std::copy(elem_partition.begin(), elem_partition.end(), std::back_inserter(elem_to_proc));
208
209 end = seacas_timer();
210 fmt::print(stderr, "\tMETIS Partition = {:.5}\n", end - start);
211 fmt::print(stderr, "Objective value = {}\n", obj_val);
212
213 // TODO Check Error...
214 }
215 }
216 #endif
217
218 void scale_decomp(std::vector<int> &elem_to_proc, int iscale, size_t num_proc)
219 {
220 // Do the scaling (integer division...)
221 if (iscale == 0) {
222 // Auto scaling was asked for. Determine max entry in `elem_to_proc` and
223 // set the scale factor.
224 auto max_proc = *std::max_element(elem_to_proc.begin(), elem_to_proc.end());
225
226 iscale = (max_proc + 1) / num_proc;
227 fmt::print(" Element Processor Map automatic scaling factor = {}\n", iscale);
228
229 if (iscale == 0) {
230 fmt::print(stderr,
231 "ERROR: Max value in element processor map is {} which is\n"
232 "\tless than the processor count ({}). Scaling values is not possible.",
233 max_proc, num_proc);
234 exit(EXIT_FAILURE);
235 }
236 }
237 std::transform(elem_to_proc.begin(), elem_to_proc.end(), elem_to_proc.begin(),
238 [iscale](int p) { return p / iscale; });
239 }
240
241 std::pair<int, std::string> extract_iscale_name(const std::string &var_name,
242 const std::string &var_type)
243 {
244 if (var_name.empty()) {
245 fmt::print(stderr, "\nERROR: No element decomposition {} specified.\n", var_type);
246 exit(EXIT_FAILURE);
247 }
248 // If the "var_name" string contains a comma, then the value
249 // following the comma is either an integer "scale" which is
250 // divided into each entry in `elem_to_proc`, or it is the
251 // string "auto" which will automatically scale all values by
252 // the *integer* "max/processorCount"
253 //
254 // NOTE: integer division with *no* rounding is used.
255 int iscale = 1;
256 auto pos = var_name.find(',');
257 if (pos != std::string::npos) {
258 // Extract the string following the comma...
259 auto scale = var_name.substr(pos + 1);
260 if (scale == "AUTO" || scale == "auto") {
261 iscale = 0;
262 }
263 else {
264 iscale = std::stoi(scale);
265 }
266 }
267 return {iscale, var_name.substr(0, pos)};
268 }
269 } // namespace
270
271 template std::vector<int> decompose_elements(const Ioss::Region &region, SystemInterface &interFace,
272 const std::vector<float> &weights, int dummy);
273 template std::vector<int> decompose_elements(const Ioss::Region &region, SystemInterface &interFace,
274 const std::vector<float> &weights, int64_t dummy);
275
276 template <typename INT>
277 std::vector<int> decompose_elements(const Ioss::Region &region, SystemInterface &interFace,
278 const std::vector<float> &weights, INT dummy)
279 {
280 progress(__func__);
281 // Populate the 'elem_to_proc' vector with a mapping from element to processor.
282
283 size_t element_count = region.get_property("element_count").get_int();
284 size_t elem_per_proc = element_count / interFace.processor_count();
285 size_t extra = element_count % interFace.processor_count();
286
287 std::vector<int> elem_to_proc;
288 elem_to_proc.reserve(element_count);
289
290 fmt::print(stderr, "\nDecomposing {} elements across {} processors using method '{}'.\n",
291 fmt::group_digits(element_count), fmt::group_digits(interFace.processor_count()),
292 interFace.decomposition_method());
293 if (interFace.lineDecomp_) {
294 fmt::print(stderr, "\tDecomposition will be modified to put element lines/chains/columns on "
295 "same processor rank\n");
296 }
297
298 if (interFace.outputDecompMap_) {
299 fmt::print(stderr, "\tDecomposition will be output to an element map named '{}'.\n",
300 interFace.decomposition_variable());
301 }
302 if (interFace.outputDecompField_) {
303 fmt::print(stderr, "\tDecomposition will be output to an element field named '{}'.\n",
304 interFace.decomposition_variable());
305 }
306 fmt::print(stderr, "\n");
307
308 if (interFace.decomposition_method() == "linear") {
309 size_t elem_beg = 0;
310 for (size_t proc = 0; proc < interFace.processor_count(); proc++) {
311 size_t add = (proc < extra) ? 1 : 0;
312 size_t elem_end = elem_beg + elem_per_proc + add;
313
314 for (size_t elem = elem_beg; elem < elem_end; elem++) {
315 elem_to_proc.push_back(proc);
316 }
317 elem_beg = elem_end;
318 }
319 }
320 else if (interFace.decomposition_method() == "scattered" ||
321 interFace.decomposition_method() == "random") {
322 // Scattered...
323 size_t proc = 0;
324 for (size_t elem = 0; elem < element_count; elem++) {
325 elem_to_proc.push_back(proc++);
326 if (proc >= interFace.processor_count()) {
327 proc = 0;
328 }
329 }
330 if (interFace.decomposition_method() == "random") {
331 // Random... Use scattered method and then random_shuffle() the vector.
332 // Ensures that each processor has correct number of elements, but
333 // they are randomly distributed.
334 std::random_device rd;
335 std::mt19937 g(rd());
336 std::shuffle(elem_to_proc.begin(), elem_to_proc.end(), g);
337 }
338 }
339
340 else if (interFace.decomposition_method() == "rcb" || interFace.decomposition_method() == "rib" ||
341 interFace.decomposition_method() == "hsfc") {
342 Ioss::DecompUtils::decompose_zoltan(
343 region, interFace.processor_count(), interFace.decomposition_method(), elem_to_proc,
344 weights, interFace.ignore_x_, interFace.ignore_y_, interFace.ignore_z_, dummy);
345 }
346
347 else if (interFace.decomposition_method() == "rb" || interFace.decomposition_method() == "kway") {
348 #if USE_METIS
349 decompose_metis(region, interFace, elem_to_proc, dummy);
350 #else
351 fmt::print(stderr, "ERROR: Metis library not enabled in this version of slice.\n"
352 " The 'rb' and 'kway' methods are not available.\n\n");
353 std::exit(1);
354 #endif
355 }
356
357 else if (interFace.decomposition_method() == "variable") {
358 auto [iscale, elem_variable] =
359 extract_iscale_name(interFace.decomposition_variable(), "variable");
360
361 // Get all element blocks and cycle through each reading the
362 // values for the processor...
363 const auto &blocks = region.get_element_blocks();
364 auto *c_region = (Ioss::Region *)(&region);
365 c_region->begin_state(1);
366 for (const auto &block : blocks) {
367 if (!block->field_exists(elem_variable)) {
368 fmt::print(stderr, "\nERROR: Element variable '{}' does not exist on block {}.\n",
369 elem_variable, block->name());
370 exit(EXIT_FAILURE);
371 }
372 std::vector<double> tmp_vals;
373 block->get_field_data(elem_variable, tmp_vals);
374 auto block_count = block->entity_count();
375 for (int64_t i = 0; i < block_count; i++) {
376 elem_to_proc.push_back((int)tmp_vals[i]);
377 }
378 }
379 scale_decomp(elem_to_proc, iscale, interFace.processor_count());
380 }
381 else if (interFace.decomposition_method() == "map") {
382 auto [iscale, map_name] = extract_iscale_name(interFace.decomposition_variable(), "map");
383
384 Ioss::DatabaseIO *db = region.get_database();
385 int exoid = db->get_file_pointer();
386
387 bool map_read = false;
388 int map_count = ex_inquire_int(exoid, EX_INQ_ELEM_MAP);
389 if (map_count > 0) {
390 int max_name_length = ex_inquire_int(exoid, EX_INQ_DB_MAX_USED_NAME_LENGTH);
391 max_name_length = max_name_length < 32 ? 32 : max_name_length;
392 char **names = get_name_array(map_count, max_name_length);
393 int error = ex_get_names(exoid, EX_ELEM_MAP, names);
394 if (error < 0) {
395 exodus_error(__LINE__);
396 }
397
398 for (int i = 0; i < map_count; i++) {
399 if (case_compare(names[i], map_name)) {
400 elem_to_proc.resize(element_count);
401 error = ex_get_num_map(exoid, EX_ELEM_MAP, i + 1, Data(elem_to_proc));
402 if (error < 0) {
403 exodus_error(__LINE__);
404 }
405 map_read = true;
406 break;
407 }
408 }
409 delete_name_array(names, map_count);
410 }
411
412 if (!map_read) {
413 fmt::print(stderr, "\nERROR: Element decomposition map '{}' could not be read from file.\n",
414 map_name);
415 exit(EXIT_FAILURE);
416 }
417
418 scale_decomp(elem_to_proc, iscale, interFace.processor_count());
419 }
420 else if (interFace.decomposition_method() == "file") {
421 // Read the element decomposition mapping from a file. The
422 // syntax of the file is an optional element count followed by
423 // the processor for this range. If the element range is
424 // omitted, then the processor applies to the next element in
425 // the sequence. All elements must be specified or an error will
426 // be raised.
427 //
428 // Example:
429 // 0
430 // 100 1
431 // 0
432 //
433 // Will assign:
434 // * element 1 to processor 0;
435 // * followed by the next 100 elements (2 to 101) to processor 1;
436 // * followed by the next element (102) to processor 0.
437 //
438 // The resulting decomposition will have 2 elements (1, 102) on
439 // processor 0 and 100 elements (2..101) on processor 1.
440
441 const std::string &filename = interFace.decomposition_file();
442 if (filename.empty()) {
443 fmt::print(stderr, "\nERROR: No element decomposition file specified.\n");
444 exit(EXIT_FAILURE);
445 }
446
447 std::ifstream decomp_file(filename, std::ios::in);
448 if (!decomp_file.good()) {
449 fmt::print(
450 stderr,
451 "\nERROR: Element decomposition file '{}' does not exist or could not be opened.\n",
452 filename);
453 exit(EXIT_FAILURE);
454 }
455
456 std::string line;
457 size_t line_num = 0;
458 while (std::getline(decomp_file, line)) {
459 line_num++;
460 // See if 1 or 2 tokens on line...
461 std::vector<std::string> tokens;
462 tokens = SLIB::tokenize(line, ", \t");
463 size_t proc = 0;
464 size_t count = 1;
465 if (tokens.empty()) {
466 break;
467 }
468 if (tokens.size() == 1) {
469 // Just a processor specification for the next element...
470 proc = std::stoi(tokens[0]);
471 elem_to_proc.push_back(proc);
472 }
473 else {
474 // Count and processor specified.
475 count = std::stoi(tokens[0]);
476 proc = std::stoi(tokens[1]);
477 }
478 if (proc > interFace.processor_count()) {
479 fmt::print(stderr,
480 "\nERROR: Invalid processor {} specified on line {} of decomposition file.\n"
481 "\tValid range is 0..{}\n",
482 fmt::group_digits(proc), fmt::group_digits(line_num),
483 fmt::group_digits(interFace.processor_count() - 1));
484 exit(EXIT_FAILURE);
485 }
486
487 if (elem_to_proc.size() + count > element_count) {
488 fmt::print(stderr,
489 "\nERROR: The processor specification on line {}"
490 " of the decomposition file results in too many elements being specified.\n"
491 "\tThe total number of elements in the model is {}\n"
492 "\tPrior to this line, {} elements were specified.\n"
493 "\tIncluding this line, {} elements will be specified.\n",
494 fmt::group_digits(line_num), fmt::group_digits(element_count),
495 fmt::group_digits(elem_to_proc.size()),
496 fmt::group_digits(elem_to_proc.size() + count));
497 exit(EXIT_FAILURE);
498 }
499
500 for (size_t i = 0; i < count; i++) {
501 elem_to_proc.push_back(proc);
502 }
503 }
504 }
505 else {
506 fmt::print(stderr, "ERROR: Unrecognized decomposition method '{}'\n\n",
507 interFace.decomposition_method());
508 exit(EXIT_FAILURE);
509 }
510
511 assert(elem_to_proc.size() == element_count);
512 return elem_to_proc;
513 }