Debian Codegraph @

src/atlas/mesh/actions/WriteLoadBalanceReport.cc

raw
1 /*
2 * (C) Copyright 2013 ECMWF.
3 *
4 * This software is licensed under the terms of the Apache Licence Version 2.0
5 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
6 * In applying this licence, ECMWF does not waive the privileges and immunities
7 * granted to it by virtue of its status as an intergovernmental organisation
8 * nor does it submit to any jurisdiction.
9 */
10
11 #include <fstream>
12 #include <iomanip>
13 #include <numeric>
14
15 #include "eckit/filesystem/PathName.h"
16
17 #include "atlas/mesh/HybridElements.h"
18 #include "atlas/mesh/IsGhostNode.h"
19 #include "atlas/mesh/Mesh.h"
20 #include "atlas/mesh/Nodes.h"
21 #include "atlas/mesh/actions/WriteLoadBalanceReport.h"
22 #include "atlas/parallel/mpi/mpi.h"
23 #include "atlas/runtime/Exception.h"
24 #include "atlas/library/FloatingPointExceptions.h"
25
26 using atlas::mesh::IsGhostNode;
27
28 namespace atlas {
29 namespace mesh {
30 namespace actions {
31
32 void write_load_balance_report(const Mesh& mesh, const std::string& filename) {
33 std::ofstream ofs;
34 if (mpi::rank() == 0) {
35 eckit::PathName path(filename);
36 ofs.open(path.localPath(), std::ofstream::out);
37 }
38
39 write_load_balance_report(mesh, ofs);
40
41 if (mpi::rank() == 0) {
42 ofs.close();
43 }
44 }
45
46 void write_load_balance_report(const Mesh& mesh, std::ostream& ofs) {
47 idx_t npart = mpi::size();
48 idx_t root = 0;
49
50 std::vector<idx_t> nb_total_nodes(npart, 0);
51 std::vector<idx_t> nb_owned_nodes(npart, 0);
52 std::vector<idx_t> nb_ghost_nodes(npart, 0);
53 std::vector<double> ghost_ratio_nodes(npart, 0);
54
55 std::vector<idx_t> nb_total_edges(npart, 0);
56 std::vector<idx_t> nb_owned_edges(npart, 0);
57 std::vector<idx_t> nb_ghost_edges(npart, 0);
58 std::vector<double> nb_ghost_ratio_edges(npart, 0);
59
60 {
61 const mesh::Nodes& nodes = mesh.nodes();
62 IsGhostNode is_ghost(nodes);
63 idx_t nb_nodes = nodes.size();
64 idx_t nowned(0);
65 idx_t nghost(0);
66 for (idx_t n = 0; n < nb_nodes; ++n) {
67 if (is_ghost(n)) {
68 ++nghost;
69 }
70 else {
71 ++nowned;
72 }
73 }
74
75 /// @note this could be improved by packing the 3 integers in a vector, and
76 /// doing only comm() call
77
78 ATLAS_TRACE_MPI(GATHER) {
79 mpi::comm().gather(nb_nodes, nb_total_nodes, root);
80 mpi::comm().gather(nowned, nb_owned_nodes, root);
81 mpi::comm().gather(nghost, nb_ghost_nodes, root);
82 }
83
84 bool disabled_fpe = library::disable_floating_point_exception(FE_INVALID);
85
86 for (idx_t p = 0; p < npart; ++p) {
87 if (nb_owned_nodes[p]) {
88 ghost_ratio_nodes[p] = static_cast<double>(nb_ghost_nodes[p]) / static_cast<double>(nb_owned_nodes[p]);
89 }
90 else {
91 ghost_ratio_nodes[p] = -1;
92 }
93 }
94 if (disabled_fpe) {
95 library::enable_floating_point_exception(FE_INVALID);
96 }
97 }
98
99 bool has_edges = mesh.edges().size();
100
101 if (has_edges) {
102 const mesh::Nodes& nodes = mesh.nodes();
103 IsGhostNode is_ghost(nodes);
104 const mesh::HybridElements::Connectivity& edge_nodes = mesh.edges().node_connectivity();
105 idx_t nb_edges = mesh.edges().size();
106 idx_t nowned(0);
107 idx_t nghost(0);
108 for (idx_t j = 0; j < nb_edges; ++j) {
109 if (is_ghost(edge_nodes(j, 0))) {
110 ++nghost;
111 }
112 else {
113 ++nowned;
114 }
115 }
116
117 /// @note this could be improved by packing the 3 integers in a vector, and
118 /// doing only comm() call
119
120 mpi::comm().gather(nb_edges, nb_total_edges, root);
121 mpi::comm().gather(nowned, nb_owned_edges, root);
122 mpi::comm().gather(nghost, nb_ghost_nodes, root);
123 }
124
125 if (mpi::rank() == 0) {
126 int idt = 10;
127 ofs << "# STATISTICS\n";
128 ofs << std::setw(1) << "#" << std::setw(5) << "";
129 ofs << std::setw(idt) << "nodes";
130 ofs << std::setw(idt) << "owned";
131 ofs << std::setw(idt) << "ghost";
132 ofs << std::setw(idt) << "ratio(%)";
133 if (has_edges) {
134 ofs << std::setw(idt) << "edges";
135 ofs << std::setw(idt) << "oedges";
136 ofs << std::setw(idt) << "gedges";
137 }
138 ofs << "\n";
139 ofs << std::setw(6) << "# tot ";
140 ofs << std::setw(idt) << std::accumulate(nb_total_nodes.data(), nb_total_nodes.data() + npart, 0);
141 ofs << std::setw(idt) << std::accumulate(nb_owned_nodes.data(), nb_owned_nodes.data() + npart, 0);
142 ofs << std::setw(idt) << std::accumulate(nb_ghost_nodes.data(), nb_ghost_nodes.data() + npart, 0);
143 ofs << std::setw(idt) << "/";
144 if (has_edges) {
145 ofs << std::setw(idt) << std::accumulate(nb_total_edges.data(), nb_total_edges.data() + npart, 0);
146 ofs << std::setw(idt) << std::accumulate(nb_owned_edges.data(), nb_owned_edges.data() + npart, 0);
147 ofs << std::setw(idt) << std::accumulate(nb_ghost_edges.data(), nb_ghost_edges.data() + npart, 0);
148 }
149 ofs << "\n";
150 ofs << std::setw(6) << "# max ";
151 ofs << std::setw(idt) << *std::max_element(nb_total_nodes.data(), nb_total_nodes.data() + npart);
152 ofs << std::setw(idt) << *std::max_element(nb_owned_nodes.data(), nb_owned_nodes.data() + npart);
153 ofs << std::setw(idt) << *std::max_element(nb_ghost_nodes.data(), nb_ghost_nodes.data() + npart);
154 ofs << std::setw(idt) << std::setw(idt) << std::fixed << std::setprecision(2)
155 << *std::max_element(ghost_ratio_nodes.data(), ghost_ratio_nodes.data() + npart) * 100.;
156 if (has_edges) {
157 ofs << std::setw(idt) << *std::max_element(nb_total_edges.data(), nb_total_edges.data() + npart);
158 ofs << std::setw(idt) << *std::max_element(nb_owned_edges.data(), nb_owned_edges.data() + npart);
159 ofs << std::setw(idt) << *std::max_element(nb_ghost_edges.data(), nb_ghost_edges.data() + npart);
160 }
161 ofs << "\n";
162 ofs << std::setw(6) << "# min ";
163 ofs << std::setw(idt) << *std::min_element(nb_total_nodes.data(), nb_total_nodes.data() + npart);
164 ofs << std::setw(idt) << *std::min_element(nb_owned_nodes.data(), nb_owned_nodes.data() + npart);
165 ofs << std::setw(idt) << *std::min_element(nb_ghost_nodes.data(), nb_ghost_nodes.data() + npart);
166 ofs << std::setw(idt) << std::fixed << std::setprecision(2)
167 << *std::min_element(ghost_ratio_nodes.data(), ghost_ratio_nodes.data() + npart) * 100.;
168 if (has_edges) {
169 ofs << std::setw(idt) << *std::min_element(nb_total_edges.data(), nb_total_edges.data() + npart);
170 ofs << std::setw(idt) << *std::min_element(nb_owned_edges.data(), nb_owned_edges.data() + npart);
171 ofs << std::setw(idt) << *std::min_element(nb_ghost_edges.data(), nb_ghost_edges.data() + npart);
172 }
173 ofs << "\n";
174 ofs << std::setw(6) << "# avg ";
175 ofs << std::setw(idt) << std::accumulate(nb_total_nodes.data(), nb_total_nodes.data() + npart, 0) / npart;
176 ofs << std::setw(idt) << std::accumulate(nb_owned_nodes.data(), nb_owned_nodes.data() + npart, 0) / npart;
177 ofs << std::setw(idt) << std::accumulate(nb_ghost_nodes.data(), nb_ghost_nodes.data() + npart, 0) / npart;
178 ofs << std::setw(idt) << std::fixed << std::setprecision(2)
179 << std::accumulate(ghost_ratio_nodes.data(), ghost_ratio_nodes.data() + npart, 0.) /
180 static_cast<double>(npart) * 100.;
181 if (has_edges) {
182 ofs << std::setw(idt) << std::accumulate(nb_total_edges.data(), nb_total_edges.data() + npart, 0) / npart;
183 ofs << std::setw(idt) << std::accumulate(nb_owned_edges.data(), nb_owned_edges.data() + npart, 0) / npart;
184 ofs << std::setw(idt) << std::accumulate(nb_ghost_edges.data(), nb_ghost_edges.data() + npart, 0) / npart;
185 }
186 ofs << "\n";
187 ofs << "#----------------------------------------------------\n";
188 ofs << "# PER TASK\n";
189 ofs << std::setw(6) << "# part";
190 ofs << std::setw(idt) << "nodes";
191 ofs << std::setw(idt) << "owned";
192 ofs << std::setw(idt) << "ghost";
193 ofs << std::setw(idt) << "ratio(%)";
194 if (has_edges) {
195 ofs << std::setw(idt) << "edges";
196 ofs << std::setw(idt) << "oedges";
197 ofs << std::setw(idt) << "gedges";
198 }
199 ofs << "\n";
200 for (idx_t jpart = 0; jpart < npart; ++jpart) {
201 ofs << std::setw(6) << jpart;
202 ofs << std::setw(idt) << nb_total_nodes[jpart];
203 ofs << std::setw(idt) << nb_owned_nodes[jpart];
204 ofs << std::setw(idt) << nb_ghost_nodes[jpart];
205 ofs << std::setw(idt) << std::fixed << std::setprecision(2) << ghost_ratio_nodes[jpart] * 100.;
206 if (has_edges) {
207 ofs << std::setw(idt) << nb_total_edges[jpart];
208 ofs << std::setw(idt) << nb_owned_edges[jpart];
209 ofs << std::setw(idt) << nb_ghost_edges[jpart];
210 }
211 ofs << "\n";
212 }
213 }
214 }
215
216 // ------------------------------------------------------------------
217
218 // C wrapper interfaces to C++ routines
219 void atlas__write_load_balance_report(Mesh::Implementation* mesh, char* filename) {
220 ATLAS_ASSERT(mesh != nullptr, "Cannot access uninitialised atlas_Mesh");
221 Mesh m(mesh);
222 write_load_balance_report(m, std::string(filename));
223 }
224
225 // ------------------------------------------------------------------
226
227 } // namespace actions
228 } // namespace mesh
229 } // namespace atlas