Debian Codegraph @

src/atlas/interpolation/AssembleGlobalMatrix.cc

raw
1 /*
2 * (C) Copyright 2025- ECMWF.
3 *
4 * This software is licensed under the terms of the Apache Licence Version 2.0
5 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
6 *
7 * In applying this licence, ECMWF does not waive the privileges and immunities
8 * granted to it by virtue of its status as an intergovernmental organisation nor
9 * does it submit to any jurisdiction.
10 */
11
12 #include "AssembleGlobalMatrix.h"
13
14 #include <unordered_map>
15
16 #include "eckit/linalg/types.h"
17
18 #include "atlas/array.h"
19 #include "atlas/linalg/sparse/SparseMatrixToTriplets.h"
20 #include "atlas/functionspace/StructuredColumns.h"
21 #include "atlas/functionspace/NodeColumns.h"
22
23 namespace atlas::interpolation {
24
25 linalg::SparseMatrixStorage assemble_global_matrix(const Interpolation& interpolation, int mpi_root) {
26 auto& mpi_comm = mpi::comm();
27
28 auto compute_max_global_index = [&mpi_comm,mpi_root](const FunctionSpace& fs) {
29 auto global_index = array::make_view<gidx_t, 1>(fs.global_index());
30 auto ghost = array::make_view<int, 1>(fs.ghost());
31 gidx_t max_gidx{0};
32 for( size_t i = 0; i < global_index.size(); ++i) {
33 if (not ghost(i)) {
34 max_gidx = std::max(max_gidx, global_index(i));
35 }
36 }
37 #if ATLAS_ECKIT_VERSION_AT_LEAST(1, 22, 2)
38 // eckit::mpi::Comm::reduceInPlace() was only added in eckit 1.22.2
39 mpi_comm.reduceInPlace(max_gidx, eckit::mpi::max(), mpi_root);
40 #else
41 mpi_comm.allReduceInPlace(max_gidx, eckit::mpi::max());
42 (void)mpi_root; // to silence unused lambda capture warning
43 #endif
44 return max_gidx;
45 };
46
47 auto src_fs = interpolation.source();
48 auto tgt_fs = interpolation.target();
49
50 // compute max column and row indices
51 gidx_t tgt_max_gidx = compute_max_global_index(tgt_fs);
52 gidx_t src_max_gidx = compute_max_global_index(src_fs);
53
54 return assemble_global_matrix(tgt_max_gidx, src_max_gidx, interpolation, mpi_root);
55 }
56
57
58 linalg::SparseMatrixStorage assemble_global_matrix(size_t nrows, size_t ncols, const Interpolation& interpolation, int mpi_root) {
59
60
61 auto src_fs = interpolation.source();
62 auto tgt_fs = interpolation.target();
63
64 auto& mpi_comm = mpi::comm();
65 auto mpi_size = mpi_comm.size();
66 auto mpi_rank = mpi_comm.rank();
67
68 std::vector<gidx_t> global_cols;
69 std::vector<gidx_t> global_rows;
70 std::vector<double> global_vals;
71
72 // Compute global_cols, global_rows, global_vals
73 {
74 std::vector<gidx_t> local_cols;
75 std::vector<gidx_t> local_rows;
76 std::vector<double> local_vals;
77
78 // Compute local_cols, local_rows, local_vals
79 {
80 interpolation::MatrixCache interpolation_cache = interpolation.createCache();
81 auto local_matrix_storage = interpolation_cache.matrix();
82 auto local_matrix = atlas::linalg::make_host_view<eckit::linalg::Scalar,eckit::linalg::Index>(local_matrix_storage);
83
84 const auto src_ridx = array::make_indexview<idx_t, 1>(src_fs.remote_index());
85
86 const auto src_global_index = array::make_view<gidx_t, 1>(src_fs.global_index());
87 const auto tgt_global_index = array::make_view<gidx_t, 1>(tgt_fs.global_index());
88 const auto src_part = array::make_view<int, 1>(src_fs.partition());
89 const auto tgt_ghost = array::make_view<int, 1>(tgt_fs.ghost());
90
91 eckit::mpi::Buffer<gidx_t> recv_global_idx_buf(mpi_size);
92 {
93 std::vector<gidx_t> send_global_idx(src_global_index.size());
94 for (int i = 0; i < send_global_idx.size(); ++i) {
95 send_global_idx[i] = src_global_index[i];
96 }
97 mpi_comm.allGatherv(send_global_idx.begin(), send_global_idx.end(), recv_global_idx_buf);
98 }
99 auto recv_global_idx = [&recv_global_idx_buf](int part, int ridx) {
100 return recv_global_idx_buf.buffer[ recv_global_idx_buf.displs[part] + ridx ];
101 };
102 auto src_gidx = [&](auto idx) {
103 return recv_global_idx(src_part(idx), src_ridx(idx));
104 };
105
106 local_cols.reserve(local_matrix.nnz());
107 local_rows.reserve(local_matrix.nnz());
108 local_vals.reserve(local_matrix.nnz());
109
110 const auto* outer = local_matrix.outer();
111 const auto* inner = local_matrix.inner();
112 const auto* value = local_matrix.value();
113 const size_t nrows = local_matrix.rows();
114
115 for(idx_t r = 0; r < nrows; ++r) {
116 if( not tgt_ghost(r) ) {
117 idx_t row = tgt_global_index(r);
118 for (idx_t c = outer[r]; c < outer[r + 1]; ++c) {
119 auto src_idx = inner[c];
120 auto col = src_gidx(src_idx);
121 auto val = value[c];
122 local_rows.emplace_back(row);
123 local_cols.emplace_back(col);
124 local_vals.emplace_back(val);
125 }
126 }
127 }
128 }
129
130 // Gather local to global on mpi_root
131 size_t local_nnz = local_vals.size();
132 std::vector<size_t> local_nnz_per_rank(mpi_size);
133 mpi_comm.gather(local_nnz, local_nnz_per_rank, mpi_root);
134
135 constexpr int mpi_tag = 0;
136 if (mpi_rank == mpi_root) {
137 size_t global_nnz = std::accumulate(local_nnz_per_rank.begin(), local_nnz_per_rank.end(), 0);
138 global_cols.resize(global_nnz);
139 global_rows.resize(global_nnz);
140 global_vals.resize(global_nnz);
141 size_t pos = 0;
142 size_t pos_root = 0;
143 for (int jproc = 0; jproc < mpi_size; ++jproc) {
144 if (jproc == mpi_root) {
145 pos += (jproc == 0) ? 0 : local_nnz_per_rank[jproc - 1];
146 pos_root = pos;
147 continue;
148 }
149 pos += (jproc == 0 ? 0 : local_nnz_per_rank[jproc - 1]);
150 mpi_comm.receive(global_cols.data() + pos, local_nnz_per_rank[jproc], jproc, mpi_tag);
151 mpi_comm.receive(global_rows.data() + pos, local_nnz_per_rank[jproc], jproc, mpi_tag);
152 mpi_comm.receive(global_vals.data() + pos, local_nnz_per_rank[jproc], jproc, mpi_tag);
153 }
154 for (int i = 0; i < local_nnz; ++i) {
155 global_cols[i + pos_root] = local_cols[i];
156 global_rows[i + pos_root] = local_rows[i];
157 global_vals[i + pos_root] = local_vals[i];
158 }
159 }
160 else {
161 mpi_comm.send(local_cols.data(), local_cols.size(), mpi_root, mpi_tag);
162 mpi_comm.send(local_rows.data(), local_rows.size(), mpi_root, mpi_tag);
163 mpi_comm.send(local_vals.data(), local_vals.size(), mpi_root, mpi_tag);
164 }
165 }
166
167 linalg::SparseMatrixStorage global_matrix;
168 if (mpi_rank == mpi_root) {
169 size_t index_base = 1;
170 bool is_sorted = false;
171 global_matrix = atlas::linalg::make_sparse_matrix_storage_from_rows_columns_values(nrows, ncols, global_rows, global_cols, global_vals, index_base, is_sorted);
172 }
173 return global_matrix;
174 }
175
176 template <typename ViewValue, typename ViewIndex, typename Value, typename Index>
177 void distribute_global_matrix(const linalg::SparseMatrixView<ViewValue,ViewIndex>& global_matrix,
178 const array::Array& partition, std::vector<Index>& rows, std::vector<Index>& cols, std::vector<Value>& vals, int mpi_root) {
179 ATLAS_TRACE("distribute_global_matrix_lowlevel");
180
181 const auto tgt_part_glb = array::make_view<int,1>(partition);
182
183 auto& mpi_comm = mpi::comm();
184 auto mpi_size = mpi_comm.size();
185 auto mpi_rank = mpi_comm.rank();
186
187 // compute how many nnz-entries each task gets
188 size_t nnz_loc = 0;
189 int mpi_tag = 0;
190 std::vector<std::size_t> nnz_per_task(mpi_size);
191 const auto outer = global_matrix.outer();
192 const auto inner = global_matrix.inner();
193 const auto value = global_matrix.value();
194 if (mpi_rank == mpi_root) {
195
196 ATLAS_ASSERT(global_matrix.rows() <= tgt_part_glb.size());
197 for(std::size_t r = 0; r < global_matrix.rows(); ++r) {
198 if (tgt_part_glb(r) >= mpi_size) {
199 ATLAS_DEBUG_VAR(r);
200 ATLAS_DEBUG_VAR(mpi_size);
201 ATLAS_DEBUG_VAR(tgt_part_glb(r));
202 ATLAS_ASSERT(tgt_part_glb(r) < nnz_per_task.size());
203 }
204 nnz_per_task[tgt_part_glb(r)] += outer[r+1] - outer[r];
205 }
206 for (int jproc = 0; jproc < mpi::comm().size(); ++jproc) {
207 if (jproc != mpi_root) {
208 mpi::comm().send(nnz_per_task.data() + jproc, 1, jproc, mpi_tag);
209 }
210 }
211 nnz_loc = nnz_per_task[mpi_root];
212 }
213 else {
214 mpi_comm.receive(&nnz_loc, 1, mpi_root, mpi_tag);
215 }
216
217 rows.resize(nnz_loc);
218 cols.resize(nnz_loc);
219 vals.resize(nnz_loc);
220
221 if (mpi_rank == mpi_root) {
222 std::vector<std::vector<Index>> send_rows(mpi_size);
223 std::vector<std::vector<Index>> send_cols(mpi_size);
224 std::vector<std::vector<Value>> send_vals(mpi_size);
225 for(std::size_t jproc=0; jproc < mpi_size; ++jproc) {
226 send_rows[jproc].reserve(nnz_per_task[jproc]);
227 send_cols[jproc].reserve(nnz_per_task[jproc]);
228 send_vals[jproc].reserve(nnz_per_task[jproc]);
229 }
230 for(std::size_t r = 0; r < global_matrix.rows(); ++r) {
231 int jproc = tgt_part_glb(r);
232 for (auto c = outer[r]; c < outer[r + 1]; ++c) {
233 auto col = inner[c];
234 send_rows[jproc].emplace_back(r);
235 send_cols[jproc].emplace_back(col);
236 send_vals[jproc].emplace_back(value[c]);
237 }
238 }
239 for(std::size_t jproc = 0; jproc < mpi_size; ++jproc) {
240 if (jproc != mpi_root) {
241 mpi_comm.send(send_rows[jproc].data(), send_rows[jproc].size(), jproc, mpi_tag);
242 mpi_comm.send(send_cols[jproc].data(), send_cols[jproc].size(), jproc, mpi_tag);
243 mpi_comm.send(send_vals[jproc].data(), send_vals[jproc].size(), jproc, mpi_tag);
244 }
245 else {
246 rows = send_rows[jproc];
247 cols = send_cols[jproc];
248 vals = send_vals[jproc];
249 }
250 }
251 }
252 else {
253 mpi_comm.receive(rows.data(), nnz_loc, mpi_root, mpi_tag);
254 mpi_comm.receive(cols.data(), nnz_loc, mpi_root, mpi_tag);
255 mpi_comm.receive(vals.data(), nnz_loc, mpi_root, mpi_tag);
256 }
257 }
258
259 linalg::SparseMatrixStorage distribute_global_matrix(const FunctionSpace& src_fs, const FunctionSpace& tgt_fs, const linalg::SparseMatrixStorage& gmatrix, int mpi_root) {
260 ATLAS_TRACE("distribute_global_matrix");
261 Field field_tgt_part_glb = tgt_fs.createField(tgt_fs.partition(), option::global(mpi_root));
262
263 // This should not be needed, but there's a problem with the NodeColumns functionspace for ORCA grids leading to uninitialized partitions
264 if (functionspace::NodeColumns(tgt_fs)) {
265 if (auto grid = functionspace::NodeColumns(tgt_fs).mesh().grid()) {
266 if (grid.type() == "ORCA") {
267 array::make_view<int,1>(field_tgt_part_glb).assign(0);
268 }
269 }
270 }
271
272 ATLAS_TRACE_SCOPE("gather partition") {
273 tgt_fs.gather(tgt_fs.partition(), field_tgt_part_glb);
274 }
275 using Index = eckit::linalg::Index;
276 using Value = eckit::linalg::Scalar;
277 std::vector<Index> rows, cols;
278 std::vector<Value> vals;
279 distribute_global_matrix(atlas::linalg::make_host_view<Value, Index>(gmatrix), field_tgt_part_glb, rows, cols, vals, mpi_root);
280
281 // map global index to local index
282 std::unordered_map<gidx_t, idx_t> to_local_rows;
283 std::unordered_map<gidx_t, idx_t> to_local_cols;
284
285 ATLAS_TRACE_SCOPE("convert to local indexing") {
286 auto tgt_gidx_exchanged = tgt_fs.createField(tgt_fs.global_index());
287 tgt_gidx_exchanged.array().copy(tgt_fs.global_index());
288 tgt_fs.haloExchange(tgt_gidx_exchanged);
289 const auto tgt_global_index = array::make_view<gidx_t, 1>(tgt_gidx_exchanged);
290 const auto tgt_ghost = array::make_view<int,1>(tgt_fs.ghost());
291
292 auto src_gidx_exchanged = src_fs.createField(src_fs.global_index());
293 src_gidx_exchanged.array().copy(src_fs.global_index());
294 src_fs.haloExchange(src_gidx_exchanged);
295 const auto src_global_index = array::make_view<gidx_t, 1>(src_gidx_exchanged);
296 const auto src_ghost = array::make_view<int,1>(src_fs.ghost());
297
298 for (idx_t r = 0; r < tgt_global_index.size(); ++r) {
299 auto gr = tgt_global_index(r);
300 if (tgt_ghost(r) && to_local_rows.find(gr) != to_local_rows.end()) {
301 continue;
302 }
303 to_local_rows[gr] = r;
304 }
305 for (idx_t c = 0; c < src_global_index.size(); ++c) {
306 auto gc = src_global_index(c);
307 if (src_ghost(c) && to_local_cols.find(gc) != to_local_cols.end()) {
308 continue;
309 }
310 to_local_cols[gc] = c;
311 }
312 for (int r = 0; r < rows.size(); ++r) {
313 rows[r] = to_local_rows[rows[r] + 1];
314 cols[r] = to_local_cols[cols[r] + 1];
315 }
316 }
317
318 linalg::SparseMatrixStorage matrix;
319 constexpr int index_base = 0;
320 constexpr bool is_sorted = false;
321 matrix = linalg::make_sparse_matrix_storage_from_rows_columns_values(tgt_fs.size(), src_fs.size(), rows, cols, vals, index_base, is_sorted);
322
323 return matrix;
324 }
325
326
327 } //end namespace
328