Debian Codegraph @

src/atlas/parallel/GatherScatter.h

raw
1 /*
2 * (C) Copyright 2013 ECMWF.
3 *
4 * This software is licensed under the terms of the Apache Licence Version 2.0
5 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
6 * In applying this licence, ECMWF does not waive the privileges and immunities
7 * granted to it by virtue of its status as an intergovernmental organisation
8 * nor does it submit to any jurisdiction.
9 */
10
11 #pragma once
12
13 #include <numeric>
14 #include <stdexcept>
15 #include <type_traits>
16 #include <vector>
17
18 #include "atlas/array/ArrayView.h"
19 #include "atlas/library/config.h"
20 #include "atlas/parallel/mpi/mpi.h"
21 #include "atlas/runtime/Exception.h"
22 #include "atlas/util/Object.h"
23
24 namespace atlas {
25 namespace parallel {
26
27 //----------------------------------------------------------------------------------------------------------------------
28
29 template <typename DATA_TYPE>
30 class Field {
31 public:
32 Field() {}
33
34 Field(const DATA_TYPE data_[], const idx_t var_strides_[], const idx_t var_shape_[], const idx_t var_rank_):
35 data(const_cast<DATA_TYPE*>(data_)), var_rank(var_rank_) {
36 var_strides.assign(var_strides_, var_strides_ + var_rank_);
37 var_shape.assign(var_shape_, var_shape_ + var_rank_);
38 }
39
40 Field(const DATA_TYPE data_[], const idx_t nb_vars): data(const_cast<DATA_TYPE*>(data_)), var_rank(1) {
41 var_strides.assign(1, 1);
42 var_shape.assign(1, nb_vars);
43 }
44
45 template <typename Value, int Rank>
46 Field(const array::ArrayView<Value, Rank>& arr) {
47 data = const_cast<DATA_TYPE*>(arr.data());
48
49 var_rank = Rank;
50 var_strides.resize(var_rank);
51 var_shape.resize(var_rank);
52
53 if (arr.rank() > 1) {
54 var_strides[0] = arr.stride(0);
55 var_shape[0] = 1;
56 for (int j = 1; j < Rank; ++j) {
57 var_strides[j] = arr.stride(j);
58 var_shape[j] = arr.shape(j);
59 }
60 }
61 else {
62 var_strides[0] = arr.stride(0);
63 var_shape[0] = 1;
64 }
65 }
66
67 template <typename Value, int Rank>
68 Field(const array::LocalView<Value, Rank>& arr) {
69 data = const_cast<DATA_TYPE*>(arr.data());
70
71 var_rank = Rank;
72 var_strides.resize(var_rank);
73 var_shape.resize(var_rank);
74
75 if (arr.rank() > 1) {
76 var_strides[0] = arr.stride(0);
77 var_shape[0] = 1;
78 for (int j = 1; j < Rank; ++j) {
79 var_strides[j] = arr.stride(j);
80 var_shape[j] = arr.shape(j);
81 }
82 }
83 else {
84 var_strides[0] = arr.stride(0);
85 var_shape[0] = 1;
86 }
87 }
88
89 public:
90 DATA_TYPE* data;
91 std::vector<idx_t> var_strides;
92 std::vector<idx_t> var_shape;
93 idx_t var_rank;
94 };
95
96 class GatherScatter : public util::Object {
97 public:
98 GatherScatter();
99 GatherScatter(const std::string& name);
100 virtual ~GatherScatter() {}
101
102 public: // methods
103 std::string name() const { return name_; }
104
105 /// @brief Setup
106 /// @param [in] mpi_comm Name of mpi communicator
107 /// @param [in] part List of partitions
108 /// @param [in] remote_idx List of local indices on remote partitions
109 /// @param [in] base values of remote_idx start at "base"
110 /// @param [in] glb_idx List of global indices
111 /// @param [in] parsize size of given lists
112 void setup(const std::string& mpi_comm, const int part[], const idx_t remote_idx[], const int base, const gidx_t glb_idx[], const idx_t parsize);
113
114 /// @brief Setup
115 /// @param [in] part List of partitions
116 /// @param [in] remote_idx List of local indices on remote partitions
117 /// @param [in] base values of remote_idx start at "base"
118 /// @param [in] glb_idx List of global indices
119 /// @param [in] parsize size of given lists
120 void setup(const int part[], const idx_t remote_idx[], const int base, const gidx_t glb_idx[], const idx_t parsize);
121
122 /// @brief Setup
123 /// @param [in] mpi_comm Name of mpi communicator
124 /// @param [in] part List of partitions
125 /// @param [in] remote_idx List of local indices on remote partitions
126 /// @param [in] base values of remote_idx start at "base"
127 /// @param [in] glb_idx List of global indices
128 /// @param [in] parsize size of given lists
129 /// @param [in] mask Mask indices not to include in the communication
130 /// pattern (0=include,1=exclude)
131 void setup(const std::string& mpi_comm, const int part[], const idx_t remote_idx[], const int base, const gidx_t glb_idx[], const int mask[],
132 const idx_t parsize);
133
134 /// @brief Setup
135 /// @param [in] part List of partitions
136 /// @param [in] remote_idx List of local indices on remote partitions
137 /// @param [in] base values of remote_idx start at "base"
138 /// @param [in] glb_idx List of global indices
139 /// @param [in] parsize size of given lists
140 /// @param [in] mask Mask indices not to include in the communication
141 /// pattern (0=include,1=exclude)
142 void setup(const int part[], const idx_t remote_idx[], const int base, const gidx_t glb_idx[], const int mask[],
143 const idx_t parsize);
144
145 template <typename DATA_TYPE>
146 void gather(const DATA_TYPE ldata[], const idx_t lstrides[], const idx_t lshape[], const idx_t lrank,
147 const idx_t lmpl_idxpos[], const idx_t lmpl_rank, DATA_TYPE gdata[], const idx_t gstrides[],
148 const idx_t gshape[], const idx_t grank, const idx_t gmpl_idxpos[], const idx_t gmpl_rank,
149 const idx_t root) const;
150
151 template <typename DATA_TYPE>
152 void gather(const DATA_TYPE ldata[], const idx_t lvar_strides[], const idx_t lvar_shape[], const idx_t lvar_rank,
153 DATA_TYPE gdata[], const idx_t gvar_strides[], const idx_t gvar_shape[], const idx_t gvar_rank,
154 const idx_t root = 0) const;
155
156 template <typename DATA_TYPE>
157 void gather(parallel::Field<DATA_TYPE const> lfields[], parallel::Field<DATA_TYPE> gfields[], const idx_t nb_fields,
158 const idx_t root = 0) const;
159
160 template <typename DATA_TYPE, int LRANK, int GRANK>
161 void gather(const array::ArrayView<DATA_TYPE, LRANK>& ldata, array::ArrayView<DATA_TYPE, GRANK>& gdata,
162 const idx_t root = 0) const;
163
164 template <typename DATA_TYPE>
165 void scatter(parallel::Field<DATA_TYPE const> gfields[], parallel::Field<DATA_TYPE> lfields[],
166 const idx_t nb_fields, const idx_t root = 0) const;
167
168 template <typename DATA_TYPE>
169 void scatter(const DATA_TYPE gdata[], const idx_t gstrides[], const idx_t gshape[], const idx_t grank,
170 const idx_t gmpl_idxpos[], const idx_t gmpl_rank, DATA_TYPE ldata[], const idx_t lstrides[],
171 const idx_t lshape[], const idx_t lrank, const idx_t lmpl_idxpos[], const idx_t lmpl_rank,
172 const idx_t root) const;
173
174 template <typename DATA_TYPE>
175 void scatter(const DATA_TYPE gdata[], const idx_t gvar_strides[], const idx_t gvar_shape[], const idx_t gvar_rank,
176 DATA_TYPE ldata[], const idx_t lvar_strides[], const idx_t lvar_shape[], const idx_t lvar_rank,
177 const idx_t root = 0) const;
178
179 template <typename DATA_TYPE, int GRANK, int LRANK>
180 void scatter(const array::ArrayView<DATA_TYPE, GRANK>& gdata, array::ArrayView<DATA_TYPE, LRANK>& ldata,
181 const idx_t root = 0) const;
182
183 gidx_t glb_dof() const { return glbcnt_; }
184
185 idx_t loc_dof() const { return loccnt_; }
186
187 const mpi::Comm& comm() const { return *comm_; }
188
189 private: // methods
190 template <typename DATA_TYPE>
191 void pack_send_buffer(const parallel::Field<DATA_TYPE const>& field, const std::vector<int>& sendmap,
192 DATA_TYPE send_buffer[]) const;
193
194 template <typename DATA_TYPE>
195 void unpack_recv_buffer(const std::vector<int>& recvmap, const DATA_TYPE recv_buffer[],
196 const parallel::Field<DATA_TYPE>& field) const;
197
198 template <typename DATA_TYPE, int RANK>
199 void var_info(const array::ArrayView<DATA_TYPE, RANK>& arr, std::vector<idx_t>& varstrides,
200 std::vector<idx_t>& varshape) const;
201
202 private: // data
203 std::string name_;
204 int loccnt_;
205 int glbcnt_;
206 std::vector<int> glbcounts_;
207 std::vector<int> glbdispls_;
208
209 // glbmap describes how the global MPI buffer maps to the global field
210 // locmap describes how the local MPI buffer maps to the local field
211 // Scatter:
212 // - pack global_MPI_buffer from global field using glbmap
213 // - MPI_Scatterv( global_MPI_buffer -> local_MPI_buffer)
214 // - unpack local_MPI_buffer into local field using locmap
215 std::vector<int> locmap_;
216 std::vector<int> glbmap_;
217
218 const mpi::Comm* comm_;
219 idx_t nproc;
220 idx_t myproc;
221
222 bool is_setup_;
223
224 idx_t parsize_;
225 friend class Checksum;
226
227 size_t glb_cnt(idx_t root) const { return myproc == root ? glbcnt_ : 0; }
228 };
229
230 //--------------------------------------------------------------------------------------------------
231
232 template <typename DATA_TYPE>
233 void GatherScatter::gather(parallel::Field<DATA_TYPE const> lfields[], parallel::Field<DATA_TYPE> gfields[],
234 idx_t nb_fields, const idx_t root) const {
235 if (!is_setup_) {
236 throw_Exception("GatherScatter was not setup", Here());
237 }
238
239 for (idx_t jfield = 0; jfield < nb_fields; ++jfield) {
240 const idx_t lvar_size =
241 std::accumulate(lfields[jfield].var_shape.data(),
242 lfields[jfield].var_shape.data() + lfields[jfield].var_rank, 1, std::multiplies<idx_t>());
243 const idx_t gvar_size =
244 std::accumulate(gfields[jfield].var_shape.data(),
245 gfields[jfield].var_shape.data() + gfields[jfield].var_rank, 1, std::multiplies<idx_t>());
246 const size_t loc_size = loccnt_ * lvar_size;
247 const size_t glb_size = glb_cnt(root) * gvar_size;
248 std::vector<DATA_TYPE> loc_buffer(loc_size);
249 std::vector<DATA_TYPE> glb_buffer(glb_size);
250 std::vector<int> glb_displs(nproc);
251 std::vector<int> glb_counts(nproc);
252
253 for (idx_t jproc = 0; jproc < nproc; ++jproc) {
254 glb_counts[jproc] = glbcounts_[jproc] * gvar_size;
255 glb_displs[jproc] = glbdispls_[jproc] * gvar_size;
256 }
257
258 /// Pack
259
260 pack_send_buffer(lfields[jfield], locmap_, loc_buffer.data());
261
262 /// Gather
263
264 ATLAS_TRACE_MPI(GATHER) {
265 comm().gatherv(loc_buffer, glb_buffer, glb_counts, glb_displs, root);
266 }
267
268 /// Unpack
269 if (myproc == root)
270 unpack_recv_buffer(glbmap_, glb_buffer.data(), gfields[jfield]);
271 }
272 }
273
274 template <typename DATA_TYPE>
275 void GatherScatter::gather(const DATA_TYPE ldata[], const idx_t lvar_strides[], const idx_t lvar_shape[],
276 const idx_t lvar_rank, DATA_TYPE gdata[], const idx_t gvar_strides[],
277 const idx_t gvar_shape[], const idx_t gvar_rank, const idx_t root) const {
278 parallel::Field<DATA_TYPE const> lfield(ldata, lvar_strides, lvar_shape, lvar_rank);
279 parallel::Field<DATA_TYPE> gfield(gdata, gvar_strides, gvar_shape, gvar_rank);
280 gather(&lfield, &gfield, 1, root);
281 }
282
283 template <typename DATA_TYPE>
284 void GatherScatter::scatter(parallel::Field<DATA_TYPE const> gfields[], parallel::Field<DATA_TYPE> lfields[],
285 const idx_t nb_fields, const idx_t root) const {
286 if (!is_setup_) {
287 throw_Exception("GatherScatter was not setup", Here());
288 }
289
290 for (idx_t jfield = 0; jfield < nb_fields; ++jfield) {
291 const int lvar_size =
292 std::accumulate(lfields[jfield].var_shape.data(),
293 lfields[jfield].var_shape.data() + lfields[jfield].var_rank, 1, std::multiplies<int>());
294 const int gvar_size =
295 std::accumulate(gfields[jfield].var_shape.data(),
296 gfields[jfield].var_shape.data() + gfields[jfield].var_rank, 1, std::multiplies<int>());
297 const size_t loc_size = loccnt_ * lvar_size;
298 const size_t glb_size = glb_cnt(root) * gvar_size;
299 std::vector<DATA_TYPE> loc_buffer(loc_size);
300 std::vector<DATA_TYPE> glb_buffer(glb_size);
301 std::vector<int> glb_displs(nproc);
302 std::vector<int> glb_counts(nproc);
303
304 for (idx_t jproc = 0; jproc < nproc; ++jproc) {
305 glb_counts[jproc] = glbcounts_[jproc] * gvar_size;
306 glb_displs[jproc] = glbdispls_[jproc] * gvar_size;
307 }
308
309 /// Pack
310 if (myproc == root) {
311 pack_send_buffer(gfields[jfield], glbmap_, glb_buffer.data());
312 }
313
314 /// Scatter
315
316 ATLAS_TRACE_MPI(SCATTER) {
317 comm().scatterv(glb_buffer.begin(), glb_buffer.end(), glb_counts, glb_displs, loc_buffer.begin(),
318 loc_buffer.end(), root);
319 }
320
321 /// Unpack
322 unpack_recv_buffer(locmap_, loc_buffer.data(), lfields[jfield]);
323 }
324 }
325
326 template <typename DATA_TYPE>
327 void GatherScatter::scatter(const DATA_TYPE gdata[], const idx_t gvar_strides[], const idx_t gvar_shape[],
328 const idx_t gvar_rank, DATA_TYPE ldata[], const idx_t lvar_strides[],
329 const idx_t lvar_shape[], const idx_t lvar_rank, const idx_t root) const {
330 parallel::Field<DATA_TYPE const> gfield(gdata, gvar_strides, gvar_shape, gvar_rank);
331 parallel::Field<DATA_TYPE> lfield(ldata, lvar_strides, lvar_shape, lvar_rank);
332 scatter(&gfield, &lfield, 1, root);
333 }
334
335 template <typename DATA_TYPE>
336 void GatherScatter::pack_send_buffer(const parallel::Field<DATA_TYPE const>& field, const std::vector<int>& sendmap,
337 DATA_TYPE send_buffer[]) const {
338 const idx_t sendcnt = static_cast<idx_t>(sendmap.size());
339
340 idx_t ibuf = 0;
341 const idx_t send_stride = field.var_strides[0] * field.var_shape[0];
342
343 switch (field.var_rank) {
344 case 1:
345 for (idx_t p = 0; p < sendcnt; ++p) {
346 const idx_t pp = send_stride * sendmap[p];
347 for (idx_t i = 0; i < field.var_shape[0]; ++i) {
348 DATA_TYPE tmp = field.data[pp + i * field.var_strides[0]];
349 send_buffer[ibuf++] = tmp;
350 }
351 }
352 break;
353 case 2:
354 for (idx_t p = 0; p < sendcnt; ++p) {
355 const idx_t pp = send_stride * sendmap[p];
356 for (idx_t i = 0; i < field.var_shape[0]; ++i) {
357 const idx_t ii = pp + i * field.var_strides[0];
358 for (idx_t j = 0; j < field.var_shape[1]; ++j) {
359 send_buffer[ibuf++] = field.data[ii + j * field.var_strides[1]];
360 }
361 }
362 }
363 break;
364 case 3:
365 for (idx_t p = 0; p < sendcnt; ++p) {
366 const idx_t pp = send_stride * sendmap[p];
367 for (idx_t i = 0; i < field.var_shape[0]; ++i) {
368 const idx_t ii = pp + i * field.var_strides[0];
369 for (idx_t j = 0; j < field.var_shape[1]; ++j) {
370 const idx_t jj = ii + j * field.var_strides[1];
371 for (idx_t k = 0; k < field.var_shape[2]; ++k) {
372 send_buffer[ibuf++] = field.data[jj + k * field.var_strides[2]];
373 }
374 }
375 }
376 }
377 break;
378 default:
379 ATLAS_NOTIMPLEMENTED;
380 }
381 }
382
383 template <typename DATA_TYPE>
384 void GatherScatter::unpack_recv_buffer(const std::vector<int>& recvmap, const DATA_TYPE recv_buffer[],
385 const parallel::Field<DATA_TYPE>& field) const {
386 const idx_t recvcnt = static_cast<idx_t>(recvmap.size());
387
388 size_t ibuf = 0;
389 const idx_t recv_stride = field.var_strides[0] * field.var_shape[0];
390
391 switch (field.var_rank) {
392 case 1:
393 for (idx_t p = 0; p < recvcnt; ++p) {
394 const idx_t pp = recv_stride * recvmap[p];
395 for (idx_t i = 0; i < field.var_shape[0]; ++i) {
396 field.data[pp + i * field.var_strides[0]] = recv_buffer[ibuf++];
397 }
398 }
399 break;
400 case 2:
401 for (idx_t p = 0; p < recvcnt; ++p) {
402 const idx_t pp = recv_stride * recvmap[p];
403 for (idx_t i = 0; i < field.var_shape[0]; ++i) {
404 const idx_t ii = pp + i * field.var_strides[0];
405 for (idx_t j = 0; j < field.var_shape[1]; ++j) {
406 field.data[ii + j * field.var_strides[1]] = recv_buffer[ibuf++];
407 }
408 }
409 }
410 break;
411 case 3:
412 for (idx_t p = 0; p < recvcnt; ++p) {
413 const idx_t pp = recv_stride * recvmap[p];
414 for (idx_t i = 0; i < field.var_shape[0]; ++i) {
415 const idx_t ii = pp + i * field.var_strides[0];
416 for (idx_t j = 0; j < field.var_shape[1]; ++j) {
417 const idx_t jj = ii + j * field.var_strides[1];
418 for (idx_t k = 0; k < field.var_shape[2]; ++k) {
419 field.data[jj + k * field.var_strides[2]] = recv_buffer[ibuf++];
420 }
421 }
422 }
423 }
424 break;
425 default:
426 ATLAS_NOTIMPLEMENTED;
427 }
428 }
429
430 template <typename DATA_TYPE, int RANK>
431 void GatherScatter::var_info(const array::ArrayView<DATA_TYPE, RANK>& arr, std::vector<idx_t>& varstrides,
432 std::vector<idx_t>& varshape) const {
433 int rank = std::max(1, RANK - 1);
434 varstrides.resize(rank);
435 varshape.resize(rank);
436
437 if (RANK > 1) {
438 idx_t stride = 1;
439 for (int j = RANK - 1; j > 0; --j) {
440 varstrides[j - 1] = stride;
441 varshape[j - 1] = arr.shape(j);
442 stride *= varshape[j - 1];
443 }
444 // varstrides.assign(arr.strides()+1,arr.strides()+RANK);
445 // varshape.assign(arr.shape()+1,arr.shape()+RANK);
446 }
447 else {
448 varstrides[0] = 1;
449 varshape[0] = 1;
450 }
451 }
452
453 template <typename DATA_TYPE, int LRANK, int GRANK>
454 void GatherScatter::gather(const array::ArrayView<DATA_TYPE, LRANK>& ldata, array::ArrayView<DATA_TYPE, GRANK>& gdata,
455 const idx_t root) const {
456 if (ldata.shape(0) == parsize_ && gdata.shape(0) == idx_t(glb_cnt(root))) {
457 std::vector<parallel::Field<DATA_TYPE const>> lfields(1, parallel::Field<DATA_TYPE const>(ldata));
458 std::vector<parallel::Field<DATA_TYPE>> gfields(1, parallel::Field<DATA_TYPE>(gdata));
459 gather(lfields.data(), gfields.data(), 1, root);
460 }
461 else {
462 ATLAS_NOTIMPLEMENTED; // Need to implement with parallel ranks > 1
463 }
464 }
465
466 template <typename DATA_TYPE, int GRANK, int LRANK>
467 void GatherScatter::scatter(const array::ArrayView<DATA_TYPE, GRANK>& gdata, array::ArrayView<DATA_TYPE, LRANK>& ldata,
468 const idx_t root) const {
469 if (ldata.shape(0) == parsize_ && gdata.shape(0) == idx_t(glb_cnt(root))) {
470 std::vector<parallel::Field<DATA_TYPE const>> gfields(1, parallel::Field<DATA_TYPE const>(gdata));
471 std::vector<parallel::Field<DATA_TYPE>> lfields(1, parallel::Field<DATA_TYPE>(ldata));
472 scatter(gfields.data(), lfields.data(), 1, root);
473 }
474 else {
475 ATLAS_NOTIMPLEMENTED; // Need to implement with parallel ranks > 1
476 }
477 }
478
479 // ------------------------------------------------------------------
480 // C wrapper interfaces to C++ routines
481 extern "C" {
482 GatherScatter* atlas__GatherScatter__new();
483 void atlas__GatherScatter__delete(GatherScatter* This);
484 void atlas__GatherScatter__setup32(GatherScatter* This, int part[], idx_t remote_idx[], int base, int glb_idx[],
485 int parsize);
486 void atlas__GatherScatter__setup64(GatherScatter* This, int part[], idx_t remote_idx[], int base, long glb_idx[],
487 int parsize);
488 int atlas__GatherScatter__glb_dof(GatherScatter* This);
489 void atlas__GatherScatter__gather_int(GatherScatter* This, int ldata[], int lvar_strides[], int lvar_shape[],
490 int lvar_rank, int gdata[], int gvar_strides[], int gvar_shape[], int gvar_rank);
491 void atlas__GatherScatter__gather_long(GatherScatter* This, long ldata[], int lvar_strides[], int lvar_shape[],
492 int lvar_rank, long gdata[], int gvar_strides[], int gvar_shape[],
493 int gvar_rank);
494 void atlas__GatherScatter__gather_float(GatherScatter* This, float ldata[], int lvar_strides[], int lvar_shape[],
495 int lvar_rank, float gdata[], int gvar_strides[], int gvar_shape[],
496 int gvar_rank);
497 void atlas__GatherScatter__gather_double(GatherScatter* This, double ldata[], int lvar_strides[], int lvar_shape[],
498 int lvar_rank, double gdata[], int gvar_strides[], int gvar_shape[],
499 int gvar_rank);
500 void atlas__GatherScatter__scatter_int(GatherScatter* This, int gdata[], int gvar_strides[], int gvar_shape[],
501 int gvar_rank, int ldata[], int lvar_strides[], int lvar_shape[], int lvar_rank);
502 void atlas__GatherScatter__scatter_long(GatherScatter* This, long gdata[], int gvar_strides[], int gvar_shape[],
503 int gvar_rank, long ldata[], int lvar_strides[], int lvar_shape[],
504 int lvar_rank);
505 void atlas__GatherScatter__scatter_float(GatherScatter* This, float gdata[], int gvar_strides[], int gvar_shape[],
506 int gvar_rank, float ldata[], int lvar_strides[], int lvar_shape[],
507 int lvar_rank);
508 void atlas__GatherScatter__scatter_double(GatherScatter* This, double gdata[], int gvar_strides[], int gvar_shape[],
509 int gvar_rank, double ldata[], int lvar_strides[], int lvar_shape[],
510 int lvar_rank);
511 }
512
513 //--------------------------------------------------------------------------------------------------
514
515 // typedef GatherScatter Gather;
516
517 } // namespace parallel
518 } // namespace atlas