Debian Codegraph @

src/atlas/parallel/GatherScatter.cc

raw
1 /*
2 * (C) Copyright 2013 ECMWF.
3 *
4 * This software is licensed under the terms of the Apache Licence Version 2.0
5 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
6 * In applying this licence, ECMWF does not waive the privileges and immunities
7 * granted to it by virtue of its status as an intergovernmental organisation
8 * nor does it submit to any jurisdiction.
9 */
10
11 #include <algorithm>
12 #include <iostream>
13 #include <numeric>
14 #include <sstream>
15 #include <stdexcept>
16
17 #include "eckit/log/Bytes.h"
18
19 #include "atlas/array.h"
20 #include "atlas/array/ArrayView.h"
21 #include "atlas/parallel/GatherScatter.h"
22 #include "atlas/parallel/mpi/Statistics.h"
23 #include "atlas/runtime/Log.h"
24 #include "atlas/runtime/Trace.h"
25
26 #include "atlas/parallel/omp/sort.h"
27
28 namespace atlas {
29 namespace parallel {
30
31 namespace {
32 struct IsGhostPoint {
33 IsGhostPoint(const int mypart, const int part[], const idx_t ridx[], const idx_t remote_idx_base, const int N) {
34 part_ = part;
35 ridx_ = ridx;
36 remote_idx_base_ = remote_idx_base;
37 mypart_ = mypart;
38 }
39
40 bool operator()(idx_t idx) {
41 if (part_[idx] != mypart_) {
42 return true;
43 }
44 if (ridx_[idx] != remote_idx_base_ + idx) {
45 return true;
46 }
47 return false;
48 }
49 int mypart_;
50 const int* part_;
51 const idx_t* ridx_;
52 idx_t remote_idx_base_;
53 };
54
55 struct Node {
56 int p;
57 idx_t i;
58 gidx_t g;
59
60 Node() = default;
61 Node(gidx_t gid, int part, idx_t idx) {
62 g = gid;
63 p = part;
64 i = idx;
65 }
66
67 bool operator<(const Node& other) const { return (g < other.g); }
68
69 bool operator==(const Node& other) const { return (g == other.g); }
70 };
71
72 } // namespace
73
74 GatherScatter::GatherScatter(): name_(), is_setup_(false) {
75 }
76
77 GatherScatter::GatherScatter(const std::string& name): name_(name), is_setup_(false) {
78 }
79
80 void GatherScatter::setup(const int part[], const idx_t remote_idx[], const int remote_idx_base, const gidx_t glb_idx[],
81 const int mask[], const idx_t parsize) {
82 setup(mpi::comm().name(), part, remote_idx, remote_idx_base, glb_idx, parsize);
83 }
84
85 void GatherScatter::setup(const std::string& mpi_comm, const int part[], const idx_t remote_idx[], const int remote_idx_base, const gidx_t glb_idx[],
86 const int mask[], const idx_t parsize) {
87 ATLAS_TRACE("GatherScatter::setup");
88 comm_ = &mpi::comm(mpi_comm);
89 myproc = comm().rank();
90 nproc = comm().size();
91 parsize_ = parsize;
92
93 glbcounts_.resize(nproc);
94 glbcounts_.assign(nproc, 0);
95 glbdispls_.resize(nproc);
96 glbdispls_.assign(nproc, 0);
97
98 std::vector<gidx_t> sendnodes_gidx(parsize_);
99 std::vector<int> sendnodes_part(parsize_);
100 std::vector<idx_t> sendnodes_ridx(parsize_);
101
102 gidx_t glb_idx_base = 1;
103 loccnt_ = 0;
104 for (idx_t n = 0; n < parsize_; ++n) {
105 if (!mask[n]) {
106 sendnodes_gidx[loccnt_] = glb_idx[n];
107 sendnodes_part[loccnt_] = part[n];
108 sendnodes_ridx[loccnt_] = remote_idx[n] - remote_idx_base;
109 ++loccnt_;
110 }
111 }
112
113 ATLAS_TRACE_MPI(ALLGATHER) {
114 comm().allGather(loccnt_, glbcounts_.begin(), glbcounts_.end());
115 }
116
117 size_t glbcnt_size_t = std::accumulate(glbcounts_.begin(), glbcounts_.end(), size_t(0));
118 if (glbcnt_size_t > std::numeric_limits<int>::max()) {
119 ATLAS_THROW_EXCEPTION("Due to limitation of MPI we cannot use larger counts");
120 }
121
122 glbcnt_ = std::accumulate(glbcounts_.begin(), glbcounts_.end(), size_t(0));
123
124 glbdispls_[0] = 0;
125 for (idx_t jproc = 1; jproc < nproc; ++jproc) // start at 1
126 {
127 glbdispls_[jproc] = glbcounts_[jproc - 1] + glbdispls_[jproc - 1];
128 }
129
130 idx_t nb_recv_nodes = glbcnt_;
131 std::vector<Node> node_sort;
132
133 try {
134 node_sort.resize(nb_recv_nodes);
135 }
136 catch (const std::bad_alloc& e) {
137 ATLAS_THROW_EXCEPTION("Could not allocate node_sort with size " << eckit::Bytes(nb_recv_nodes * sizeof(Node)));
138 }
139
140 ATLAS_TRACE_SCOPE("receive nodes global index")
141 {
142 std::vector<gidx_t> recvnodes_gidx;
143 try {
144 recvnodes_gidx.resize(glbcnt_);
145 }
146 catch (const std::bad_alloc& e) {
147 ATLAS_THROW_EXCEPTION("Could not allocate recvnodes_gidx with size "
148 << eckit::Bytes(glbcnt_ * sizeof(gidx_t)));
149 }
150 ATLAS_TRACE_MPI(ALLGATHER) {
151 comm().allGatherv(sendnodes_gidx.begin(), sendnodes_gidx.begin() + loccnt_, recvnodes_gidx.data(),
152 glbcounts_.data(), glbdispls_.data());
153 }
154 atlas_omp_parallel_for(idx_t n = 0; n < nb_recv_nodes; ++n) {
155 node_sort[n].g = recvnodes_gidx[n];
156 }
157 sendnodes_gidx.clear();
158 }
159
160
161 ATLAS_TRACE_SCOPE("receive nodes partition")
162 {
163 std::vector<int> recvnodes_part;
164 try {
165 recvnodes_part.resize(glbcnt_);
166 }
167 catch (const std::bad_alloc& e) {
168 ATLAS_THROW_EXCEPTION("Could not allocate recvnodes_part with size "
169 << eckit::Bytes(glbcnt_ * sizeof(int)));
170 }
171 ATLAS_TRACE_MPI(ALLGATHER) {
172 comm().allGatherv(sendnodes_part.begin(), sendnodes_part.begin() + loccnt_, recvnodes_part.data(),
173 glbcounts_.data(), glbdispls_.data());
174 }
175 atlas_omp_parallel_for(idx_t n = 0; n < nb_recv_nodes; ++n) {
176 node_sort[n].p = recvnodes_part[n];
177 }
178 sendnodes_part.clear();
179 }
180
181 ATLAS_TRACE_SCOPE("receive nodes remote index")
182 {
183 std::vector<idx_t> recvnodes_ridx;
184 try {
185 recvnodes_ridx.resize(glbcnt_);
186 }
187 catch (const std::bad_alloc& e) {
188 ATLAS_THROW_EXCEPTION("Could not allocate recvnodes_ridx with size "
189 << eckit::Bytes(glbcnt_ * sizeof(idx_t)));
190 }
191 ATLAS_TRACE_MPI(ALLGATHER) {
192 comm().allGatherv(sendnodes_ridx.begin(), sendnodes_ridx.begin() + loccnt_, recvnodes_ridx.data(),
193 glbcounts_.data(), glbdispls_.data());
194 }
195 atlas_omp_parallel_for(idx_t n = 0; n < nb_recv_nodes; ++n) {
196 node_sort[n].i = recvnodes_ridx[n];
197 }
198 sendnodes_ridx.clear();
199 }
200
201 bool use_sorted_nodes = false; // `use_sorted_nodes = true` is the behaviour as it has been with atlas versions <= 0.42.0
202
203 if (use_sorted_nodes) {
204 // Sort on "g" member, and remove duplicates
205 ATLAS_TRACE_SCOPE("sorting") {
206 // omp::sort(node_sort.begin(), node_sort.end());
207 std::sort(node_sort.begin(), node_sort.end());
208 }
209 ATLAS_TRACE_SCOPE("remove duplicates") {
210 node_sort.erase(std::unique(node_sort.begin(), node_sort.end()), node_sort.end());
211 }
212 }
213
214 glbcounts_.assign(nproc, 0);
215 glbdispls_.assign(nproc, 0);
216
217 for (size_t n = 0; n < node_sort.size(); ++n) {
218 ++glbcounts_[node_sort[n].p];
219 }
220
221 glbdispls_[0] = 0;
222
223 for (idx_t jproc = 1; jproc < nproc; ++jproc) // start at 1
224 {
225 glbdispls_[jproc] = glbcounts_[jproc - 1] + glbdispls_[jproc - 1];
226 }
227
228 glbcnt_ = std::accumulate(glbcounts_.begin(), glbcounts_.end(), size_t(0));
229 loccnt_ = glbcounts_[myproc];
230
231 glbmap_.clear();
232 glbmap_.resize(glbcnt_);
233 locmap_.clear();
234 locmap_.resize(loccnt_,-1);
235 std::vector<int> idx(nproc, 0);
236
237 gidx_t n{node_sort.begin()->g - 1};
238 for (const auto& node : node_sort) {
239 glbmap_[glbdispls_[node.p] + idx[node.p]] = (use_sorted_nodes ? n++ : node.g - glb_idx_base);
240
241 if (node.p == myproc) {
242 locmap_[idx[node.p]] = node.i;
243 }
244
245 ++idx[node.p];
246 }
247
248 is_setup_ = true;
249 }
250
251 void GatherScatter::setup(const int part[], const idx_t remote_idx[], const int remote_idx_base, const gidx_t glb_idx[],
252 const idx_t parsize) {
253 setup(mpi::comm().name(), part, remote_idx, remote_idx_base, glb_idx, parsize);
254 }
255
256 void GatherScatter::setup(const std::string& mpi_comm, const int part[], const idx_t remote_idx[], const int remote_idx_base, const gidx_t glb_idx[],
257 const idx_t parsize) {
258 std::vector<int> mask(parsize);
259 {
260 int mypart = mpi::comm(mpi_comm).rank();
261 IsGhostPoint is_ghost(mypart, part, remote_idx, remote_idx_base, parsize);
262 for (idx_t jj = 0; jj < parsize; ++jj) {
263 mask[jj] = is_ghost(jj) ? 1 : 0;
264 }
265 }
266 setup(mpi_comm, part, remote_idx, remote_idx_base, glb_idx, mask.data(), parsize);
267 }
268
269 /////////////////////
270
271 GatherScatter* atlas__GatherScatter__new() {
272 return new GatherScatter();
273 }
274
275 void atlas__GatherScatter__delete(GatherScatter* This) {
276 delete This;
277 }
278
279 void atlas__GatherScatter__setup32(GatherScatter* This, int part[], idx_t remote_idx[], int remote_idx_base, int glb_idx[],
280 int parsize) {
281 #if ATLAS_BITS_GLOBAL == 32
282 This->setup(part, remote_idx, remote_idx_base, glb_idx, parsize);
283 #else
284 std::vector<gidx_t> glb_idx_convert(parsize);
285 for (int j = 0; j < parsize; ++j) {
286 glb_idx_convert[j] = glb_idx[j];
287 }
288 This->setup(part, remote_idx, remote_idx_base, glb_idx_convert.data(), parsize);
289 #endif
290 }
291
292 void atlas__GatherScatter__setup64(GatherScatter* This, int part[], idx_t remote_idx[], int remote_idx_base, long glb_idx[],
293 int parsize) {
294 #if ATLAS_BITS_GLOBAL == 64
295 This->setup(part, remote_idx, remote_idx_base, glb_idx, parsize);
296 #else
297 std::vector<gidx_t> glb_idx_convert(parsize);
298 for (idx_t j = 0; j < parsize; ++j) {
299 glb_idx_convert[j] = glb_idx[j];
300 }
301 This->setup(part, remote_idx, remote_idx_base, glb_idx_convert.data(), parsize);
302 #endif
303 }
304
305 int atlas__GatherScatter__glb_dof(GatherScatter* This) {
306 return This->glb_dof();
307 }
308
309 void atlas__GatherScatter__gather_int(GatherScatter* This, int lfield[], int lvar_strides[], int lvar_extents[],
310 int lvar_rank, int gfield[], int gvar_strides[], int gvar_extents[],
311 int gvar_rank) {
312 std::vector<idx_t> lvstrides(lvar_rank);
313 std::vector<idx_t> lvextents(lvar_rank);
314 std::vector<idx_t> gvstrides(gvar_rank);
315 std::vector<idx_t> gvextents(gvar_rank);
316 for (int n = 0; n < lvar_rank; ++n) {
317 lvstrides[n] = lvar_strides[n];
318 lvextents[n] = lvar_extents[n];
319 }
320 for (int n = 0; n < gvar_rank; ++n) {
321 gvstrides[n] = gvar_strides[n];
322 gvextents[n] = gvar_extents[n];
323 }
324 This->gather(lfield, lvstrides.data(), lvextents.data(), lvar_rank, gfield, gvstrides.data(), gvextents.data(),
325 gvar_rank);
326 }
327
328 void atlas__GatherScatter__gather_long(GatherScatter* This, long lfield[], int lvar_strides[], int lvar_extents[],
329 int lvar_rank, long gfield[], int gvar_strides[], int gvar_extents[],
330 int gvar_rank) {
331 std::vector<idx_t> lvstrides(lvar_rank);
332 std::vector<idx_t> lvextents(lvar_rank);
333 std::vector<idx_t> gvstrides(gvar_rank);
334 std::vector<idx_t> gvextents(gvar_rank);
335 for (int n = 0; n < lvar_rank; ++n) {
336 lvstrides[n] = lvar_strides[n];
337 lvextents[n] = lvar_extents[n];
338 }
339 for (int n = 0; n < gvar_rank; ++n) {
340 gvstrides[n] = gvar_strides[n];
341 gvextents[n] = gvar_extents[n];
342 }
343 This->gather(lfield, lvstrides.data(), lvextents.data(), lvar_rank, gfield, gvstrides.data(), gvextents.data(),
344 gvar_rank);
345 }
346
347 void atlas__GatherScatter__gather_float(GatherScatter* This, float lfield[], int lvar_strides[], int lvar_extents[],
348 int lvar_rank, float gfield[], int gvar_strides[], int gvar_extents[],
349 int gvar_rank) {
350 std::vector<idx_t> lvstrides(lvar_rank);
351 std::vector<idx_t> lvextents(lvar_rank);
352 std::vector<idx_t> gvstrides(gvar_rank);
353 std::vector<idx_t> gvextents(gvar_rank);
354 for (int n = 0; n < lvar_rank; ++n) {
355 lvstrides[n] = lvar_strides[n];
356 lvextents[n] = lvar_extents[n];
357 }
358 for (int n = 0; n < gvar_rank; ++n) {
359 gvstrides[n] = gvar_strides[n];
360 gvextents[n] = gvar_extents[n];
361 }
362 This->gather(lfield, lvstrides.data(), lvextents.data(), lvar_rank, gfield, gvstrides.data(), gvextents.data(),
363 gvar_rank);
364 }
365
366 void atlas__GatherScatter__gather_double(GatherScatter* This, double lfield[], int lvar_strides[], int lvar_extents[],
367 int lvar_rank, double gfield[], int gvar_strides[], int gvar_extents[],
368 int gvar_rank) {
369 std::vector<idx_t> lvstrides(lvar_rank);
370 std::vector<idx_t> lvextents(lvar_rank);
371 std::vector<idx_t> gvstrides(gvar_rank);
372 std::vector<idx_t> gvextents(gvar_rank);
373 for (int n = 0; n < lvar_rank; ++n) {
374 lvstrides[n] = lvar_strides[n];
375 lvextents[n] = lvar_extents[n];
376 }
377 for (int n = 0; n < gvar_rank; ++n) {
378 gvstrides[n] = gvar_strides[n];
379 gvextents[n] = gvar_extents[n];
380 }
381 This->gather(lfield, lvstrides.data(), lvextents.data(), lvar_rank, gfield, gvstrides.data(), gvextents.data(),
382 gvar_rank);
383 }
384
385 void atlas__GatherScatter__scatter_int(GatherScatter* This, int gfield[], int gvar_strides[], int gvar_extents[],
386 int gvar_rank, int lfield[], int lvar_strides[], int lvar_extents[],
387 int lvar_rank) {
388 std::vector<idx_t> lvstrides(lvar_rank);
389 std::vector<idx_t> lvextents(lvar_rank);
390 std::vector<idx_t> gvstrides(gvar_rank);
391 std::vector<idx_t> gvextents(gvar_rank);
392 for (int n = 0; n < lvar_rank; ++n) {
393 lvstrides[n] = lvar_strides[n];
394 lvextents[n] = lvar_extents[n];
395 }
396 for (int n = 0; n < gvar_rank; ++n) {
397 gvstrides[n] = gvar_strides[n];
398 gvextents[n] = gvar_extents[n];
399 }
400 This->scatter(gfield, gvstrides.data(), gvextents.data(), gvar_rank, lfield, lvstrides.data(), lvextents.data(),
401 lvar_rank);
402 }
403
404 void atlas__GatherScatter__scatter_long(GatherScatter* This, long gfield[], int gvar_strides[], int gvar_extents[],
405 int gvar_rank, long lfield[], int lvar_strides[], int lvar_extents[],
406 int lvar_rank) {
407 std::vector<idx_t> lvstrides(lvar_rank);
408 std::vector<idx_t> lvextents(lvar_rank);
409 std::vector<idx_t> gvstrides(gvar_rank);
410 std::vector<idx_t> gvextents(gvar_rank);
411 for (int n = 0; n < lvar_rank; ++n) {
412 lvstrides[n] = lvar_strides[n];
413 lvextents[n] = lvar_extents[n];
414 }
415 for (int n = 0; n < gvar_rank; ++n) {
416 gvstrides[n] = gvar_strides[n];
417 gvextents[n] = gvar_extents[n];
418 }
419 This->scatter(gfield, gvstrides.data(), gvextents.data(), gvar_rank, lfield, lvstrides.data(), lvextents.data(),
420 lvar_rank);
421 }
422
423 void atlas__GatherScatter__scatter_float(GatherScatter* This, float gfield[], int gvar_strides[], int gvar_extents[],
424 int gvar_rank, float lfield[], int lvar_strides[], int lvar_extents[],
425 int lvar_rank) {
426 std::vector<idx_t> lvstrides(lvar_rank);
427 std::vector<idx_t> lvextents(lvar_rank);
428 std::vector<idx_t> gvstrides(gvar_rank);
429 std::vector<idx_t> gvextents(gvar_rank);
430 for (int n = 0; n < lvar_rank; ++n) {
431 lvstrides[n] = lvar_strides[n];
432 lvextents[n] = lvar_extents[n];
433 }
434 for (int n = 0; n < gvar_rank; ++n) {
435 gvstrides[n] = gvar_strides[n];
436 gvextents[n] = gvar_extents[n];
437 }
438 This->scatter(gfield, gvstrides.data(), gvextents.data(), gvar_rank, lfield, lvstrides.data(), lvextents.data(),
439 lvar_rank);
440 }
441
442 void atlas__GatherScatter__scatter_double(GatherScatter* This, double gfield[], int gvar_strides[], int gvar_extents[],
443 int gvar_rank, double lfield[], int lvar_strides[], int lvar_extents[],
444 int lvar_rank) {
445 std::vector<idx_t> lvstrides(lvar_rank);
446 std::vector<idx_t> lvextents(lvar_rank);
447 std::vector<idx_t> gvstrides(gvar_rank);
448 std::vector<idx_t> gvextents(gvar_rank);
449 for (int n = 0; n < lvar_rank; ++n) {
450 lvstrides[n] = lvar_strides[n];
451 lvextents[n] = lvar_extents[n];
452 }
453 for (int n = 0; n < gvar_rank; ++n) {
454 gvstrides[n] = gvar_strides[n];
455 gvextents[n] = gvar_extents[n];
456 }
457 This->scatter(gfield, gvstrides.data(), gvextents.data(), gvar_rank, lfield, lvstrides.data(), lvextents.data(),
458 lvar_rank);
459 }
460
461 /////////////////////
462
463 } // namespace parallel
464 } // namespace atlas