Debian Codegraph @

packages/seacas/applications/zellij/Decompose.C

raw
1 // Copyright(C) 2021, 2022, 2023 National Technology & Engineering Solutions
2 // of Sandia, LLC (NTESS). Under the terms of Contract DE-NA0003525 with
3 // NTESS, the U.S. Government retains certain rights in this software.
4 //
5 // See packages/seacas/LICENSE for details
6
7 #ifdef USE_ZOLTAN
8 #include "zoltan.h" // for Zoltan_Set_Param, etc
9 #include "zoltan_types.h" // for ZOLTAN_ID_PTR*, ZOLTAN_OK, etc
10 #include <mpi.h> // for MPI_Finalize, etc
11 #endif
12
13 #include <algorithm>
14 #include <fmt/format.h>
15 #include <random>
16 #include <vector>
17
18 #include "Cell.h"
19 #include "Grid.h"
20 #include "Ioss_ElementBlock.h"
21 #include "Ioss_ParallelUtils.h"
22 #include "Ioss_Sort.h"
23
24 extern unsigned int debug_level;
25
26 namespace {
27 #ifdef USE_ZOLTAN
28 #define STRINGIFY(x) #x
29 #define TOSTRING(x) STRINGIFY(x)
30 #define ZCHECK(funcall) \
31 do { \
32 ierr = (funcall); \
33 if (ierr == ZOLTAN_FATAL) { \
34 fmt::print(stderr, "Error returned from {} ({}:{})\n", TOSTRING(funcall), __FILE__, \
35 __LINE__); \
36 goto End; \
37 } \
38 } while (0)
39
40 /*****************************************************************************/
41 /***** Global data structure used by Zoltan callbacks. *****/
42 /***** Could implement Zoltan callbacks without global data structure, *****/
43 /***** but using the global data structure makes implementation quick. *****/
44 struct
45 {
46 size_t ndot; /* Length of x, y, z, and part (== # of elements) */
47 int *vwgt; /* vertex weights */
48 float *x; /* x-coordinates */
49 float *y; /* y-coordinates */
50 float *z; /* z-coordinates */
51 } Zoltan_Data;
52
53 /*****************************************************************************/
54 /***** ZOLTAN CALLBACK FUNCTIONS *****/
55 int zoltan_num_dim(void * /*data*/, int *ierr)
56 {
57 /* Return dimensionality of coordinate data.
58 * Using global data structure Zoltan_Data, initialized in ZOLTAN_RCB_assign.
59 */
60 *ierr = ZOLTAN_OK;
61 if (Zoltan_Data.z != nullptr) {
62 return 3;
63 }
64 if (Zoltan_Data.y != nullptr) {
65 return 2;
66 }
67 return 1;
68 }
69
70 int zoltan_num_obj(void * /*data*/, int *ierr)
71 {
72 /* Return number of objects.
73 * Using global data structure Zoltan_Data, initialized in ZOLTAN_RCB_assign.
74 */
75 *ierr = ZOLTAN_OK;
76 return Zoltan_Data.ndot;
77 }
78
79 void zoltan_obj_list(void * /*data*/, int /*ngid_ent*/, int /*nlid_ent*/, ZOLTAN_ID_PTR gids,
80 ZOLTAN_ID_PTR /*lids*/, int wdim, float *wgts, int *ierr)
81 {
82 /* Return list of object IDs.
83 * Return only global IDs; don't need local IDs since running in serial.
84 * gids are array indices for coordinate and vwgts arrays.
85 * Using global data structure Zoltan_Data, initialized in ZOLTAN_RCB_assign.
86 */
87 for (size_t i = 0; i < Zoltan_Data.ndot; i++) {
88 gids[i] = i;
89 if (wdim != 0) {
90 wgts[i] = static_cast<float>(Zoltan_Data.vwgt[i]);
91 }
92 }
93
94 *ierr = ZOLTAN_OK;
95 }
96
97 void zoltan_geom(void * /*data*/, int /*ngid_ent*/, int /*nlid_ent*/, int nobj,
98 const ZOLTAN_ID_PTR gids, ZOLTAN_ID_PTR /*lids*/, int ndim, double *geom,
99 int *ierr)
100 {
101 /* Return coordinates for objects.
102 * gids are array indices for coordinate arrays.
103 * Using global data structure Zoltan_Data, initialized in ZOLTAN_RCB_assign.
104 */
105
106 for (int i = 0; i < nobj; i++) {
107 size_t j = gids[i];
108 geom[i * ndim] = Zoltan_Data.x[j];
109 if (ndim > 1) {
110 geom[i * ndim + 1] = Zoltan_Data.y[j];
111 }
112 if (ndim > 2) {
113 geom[i * ndim + 2] = Zoltan_Data.z[j];
114 }
115 }
116
117 *ierr = ZOLTAN_OK;
118 }
119 #endif
120 } // namespace
121
122 void decompose_grid(Grid &grid, int ranks, const std::string &method)
123 {
124 if (ranks == 1) {
125 return;
126 }
127
128 if (method == "CYCLIC" || method == "LINEAR" || method == "RANDOM") {
129 std::vector<int> rank_vec(grid.size());
130
131 // Generate a "cyclic" vector 0, 1, 2, ... ranks-1, 0, 1, ...
132 std::generate(rank_vec.begin(), rank_vec.end(),
133 [n = 0, ranks]() mutable { return n++ % ranks; });
134
135 if (method == "LINEAR") {
136 Ioss::sort(rank_vec.begin(), rank_vec.end());
137 }
138 else if (method == "RANDOM") {
139 unsigned seed = std::chrono::system_clock::now().time_since_epoch().count();
140 std::shuffle(rank_vec.begin(), rank_vec.end(), std::default_random_engine(seed));
141 }
142
143 size_t k = 0;
144 for (size_t j = 0; j < grid.JJ(); j++) {
145 for (size_t i = 0; i < grid.II(); i++) {
146 auto &cell = grid.get_cell(i, j);
147 cell.set_rank(Loc::C, rank_vec[k++]);
148 }
149 }
150 return;
151 }
152
153 #ifdef USE_ZOLTAN
154 // Below here are Zoltan decompositions...
155 std::vector<float> x(grid.size());
156 std::vector<float> y(grid.size());
157 std::vector<int> w(grid.size());
158
159 // Get coordinates of each cell centroid... (actually origin of cell)
160 size_t idx = 0;
161 for (size_t j = 0; j < grid.JJ(); j++) {
162 for (size_t i = 0; i < grid.II(); i++) {
163 const auto &cell = grid.get_cell(i, j);
164 x[idx] = cell.m_offX;
165 y[idx] = cell.m_offY;
166
167 const auto &element_blocks = cell.region()->get_element_blocks();
168 for (const auto *block : element_blocks) {
169 w[idx] += block->entity_count();
170 }
171 idx++;
172 }
173 }
174
175 /* Copy mesh data and pointers into structure accessible from callback fns. */
176 Zoltan_Data.ndot = grid.size();
177 Zoltan_Data.vwgt = Data(w);
178 Zoltan_Data.x = Data(x);
179 Zoltan_Data.y = Data(y);
180 Zoltan_Data.z = nullptr;
181
182 /* Initialize Zoltan */
183 int argc = 0;
184 char **argv = nullptr;
185 ZOLTAN_ID_PTR zgids = nullptr;
186 ZOLTAN_ID_PTR zlids = nullptr; /* Useful output from Zoltan_LB_Partition */
187 int *zprocs = nullptr; /* Useful output from Zoltan_LB_Partition */
188 int *zparts = nullptr; /* Useful output from Zoltan_LB_Partition */
189
190 int ierr = 0;
191 float ver = 0.0;
192 Zoltan_Initialize(argc, argv, &ver);
193 struct Zoltan_Struct *zz = Zoltan_Create(Ioss::ParallelUtils::comm_world());
194
195 /* Register Callback functions */
196 /* Using global Zoltan_Data; could register it here instead as data field. */
197 ZCHECK(Zoltan_Set_Fn(zz, ZOLTAN_NUM_GEOM_FN_TYPE,
198 reinterpret_cast<ZOLTAN_VOID_FN *>(zoltan_num_dim), nullptr));
199 ZCHECK(Zoltan_Set_Fn(zz, ZOLTAN_NUM_OBJ_FN_TYPE,
200 reinterpret_cast<ZOLTAN_VOID_FN *>(zoltan_num_obj), nullptr));
201 ZCHECK(Zoltan_Set_Fn(zz, ZOLTAN_OBJ_LIST_FN_TYPE,
202 reinterpret_cast<ZOLTAN_VOID_FN *>(zoltan_obj_list), nullptr));
203 ZCHECK(Zoltan_Set_Fn(zz, ZOLTAN_GEOM_MULTI_FN_TYPE,
204 reinterpret_cast<ZOLTAN_VOID_FN *>(zoltan_geom), nullptr));
205
206 /* Set parameters for Zoltan */
207 ZCHECK(Zoltan_Set_Param(zz, "DEBUG_LEVEL", "0"));
208 {
209 std::string str = fmt::format("{}", ranks);
210 ZCHECK(Zoltan_Set_Param(zz, "NUM_GLOBAL_PARTITIONS", str.c_str()));
211 ZCHECK(Zoltan_Set_Param(zz, "LB_METHOD", method.c_str()));
212 }
213 ZCHECK(Zoltan_Set_Param(zz, "NUM_LID_ENTRIES", "0"));
214 ZCHECK(Zoltan_Set_Param(zz, "REMAP", "0"));
215 ZCHECK(Zoltan_Set_Param(zz, "RETURN_LISTS", "PARTITION_ASSIGNMENTS"));
216 if (Zoltan_Data.vwgt != nullptr) {
217 ZCHECK(Zoltan_Set_Param(zz, "OBJ_WEIGHT_DIM", "1"));
218 }
219 ZCHECK(Zoltan_Set_Param(zz, "RCB_RECTILINEAR_BLOCKS", "1"));
220
221 /* Call partitioner */
222 {
223 fmt::print(" Using Zoltan version {:.2}, method {}\n", static_cast<double>(ver), method);
224 int zngid_ent = 0;
225 int znlid_ent = 0; /* Useful output from Zoltan_LB_Partition */
226 int znobj = 0;
227 ZOLTAN_ID_PTR dummy1 = nullptr;
228 ZOLTAN_ID_PTR dummy2 = nullptr; /* Empty output from Zoltan_LB_Partition */
229 int dummy0 = 0;
230 int *dummy3 = nullptr;
231 int *dummy4 = nullptr;
232 int changes = 0;
233
234 ZCHECK(Zoltan_LB_Partition(zz, &changes, &zngid_ent, &znlid_ent, &dummy0, &dummy1, &dummy2,
235 &dummy3, &dummy4, &znobj, &zgids, &zlids, &zprocs, &zparts));
236
237 /* Sanity check */
238 if (grid.size() != static_cast<size_t>(znobj)) {
239 fmt::print(stderr, "Sanity check failed; ndot {} != znobj {}.\n", grid.size(),
240 static_cast<size_t>(znobj));
241 goto End;
242 }
243
244 idx = 0;
245 for (size_t j = 0; j < grid.JJ(); j++) {
246 for (size_t i = 0; i < grid.II(); i++) {
247 auto &cell = grid.get_cell(i, j);
248 cell.set_rank(Loc::C, zparts[idx++]);
249 }
250 }
251 }
252
253 End:
254 /* Clean up */
255 Zoltan_LB_Free_Part(&zgids, &zlids, &zprocs, &zparts);
256 Zoltan_Destroy(&zz);
257 if (ierr != 0) {
258 MPI_Finalize();
259 exit(-1);
260 }
261 #endif
262 }