Debian Codegraph @

src/atlas/parallel/HaloExchange.h

raw
1 /*
2 * (C) Copyright 2013 ECMWF.
3 *
4 * This software is licensed under the terms of the Apache Licence Version 2.0
5 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
6 * In applying this licence, ECMWF does not waive the privileges and immunities
7 * granted to it by virtue of its status as an intergovernmental organisation
8 * nor does it submit to any jurisdiction.
9 */
10
11 /// @file HaloExchange.h
12 /// @author Willem Deconinck
13 /// @date Nov 2013
14
15 #pragma once
16
17 #include <stdexcept>
18 #include <string>
19 #include <vector>
20
21 #include "atlas/parallel/detail/Packer.h"
22 #include "atlas/parallel/mpi/Statistics.h"
23 #include "atlas/parallel/mpi/mpi.h"
24
25 #include "atlas/array/ArrayView.h"
26 #include "atlas/array/ArrayViewDefs.h"
27 #include "atlas/array/ArrayViewUtil.h"
28 #include "atlas/array/SVector.h"
29 #include "atlas/array_fwd.h"
30 #include "atlas/library/config.h"
31 #include "atlas/library/defines.h"
32 #include "atlas/runtime/Exception.h"
33 #include "atlas/runtime/Log.h"
34 #include "atlas/util/Object.h"
35
36 namespace atlas {
37 namespace parallel {
38
39 class HaloExchange : public util::Object {
40 public:
41 HaloExchange();
42 HaloExchange(const std::string& name);
43
44 virtual ~HaloExchange();
45
46 public: // methods
47 const std::string& name() const { return name_; }
48
49 void setup(const int part[], const idx_t remote_idx[], const int base, idx_t size);
50 void setup(const std::string& mpi_comm, const int part[], const idx_t remote_idx[], const int base, idx_t size);
51
52 void setup(const int part[], const idx_t remote_idx[], const int base, idx_t size, idx_t halo_begin);
53 void setup(const std::string& mpi_comm, const int part[], const idx_t remote_idx[], const int base, idx_t size, idx_t halo_begin);
54
55 template <typename DATA_TYPE, int RANK, typename ParallelDim = array::FirstDim>
56 void execute(array::Array& field, bool on_device = false) const;
57
58 template <typename DATA_TYPE, int RANK, typename ParallelDim = array::FirstDim>
59 void execute_adjoint(array::Array& field, bool on_device = false) const;
60
61 private: // methods
62 idx_t index(idx_t i, idx_t j, idx_t k, idx_t ni, idx_t nj, idx_t /*nk*/) const { return (i + ni * (j + nj * k)); }
63
64 idx_t index(idx_t i, idx_t j, idx_t ni, idx_t /*nj*/) const { return (i + ni * j); }
65
66 template <typename DATA_TYPE>
67 void counts_displs_setup(const idx_t var_size, std::vector<int>& send_counts_init,
68 std::vector<int>& recv_counts_init, std::vector<int>& send_counts,
69 std::vector<int>& recv_counts, std::vector<int>& send_displs,
70 std::vector<int>& recv_displs) const;
71
72
73 template <typename DATA_TYPE>
74 void ireceive(int tag, std::vector<int>& recv_displs, std::vector<int>& recv_counts,
75 std::vector<eckit::mpi::Request>& recv_req, DATA_TYPE* recv_buffer) const;
76
77 template <typename DATA_TYPE>
78 void isend_and_wait_for_receive(int tag, std::vector<int>& recv_counts_init,
79 std::vector<eckit::mpi::Request>& recv_req, std::vector<int>& send_displs,
80 std::vector<int>& send_counts, std::vector<eckit::mpi::Request>& send_req,
81 DATA_TYPE* send_buffer) const;
82
83 void wait_for_send(std::vector<int>& send_counts, std::vector<eckit::mpi::Request>& send_req) const;
84
85 template <typename DATA_TYPE>
86 DATA_TYPE* allocate_buffer(const int buffer_size, const bool on_device) const;
87
88 template <typename DATA_TYPE>
89 void deallocate_buffer(DATA_TYPE* buffer, const int buffer_size, const bool on_device) const;
90
91 template <int ParallelDim, typename DATA_TYPE, int RANK>
92 void pack_send_buffer(const array::ArrayView<DATA_TYPE, RANK>& array,
93 DATA_TYPE* send_buffer, int send_buffer_size,
94 const bool on_device) const;
95
96 template <int ParallelDim, typename DATA_TYPE, int RANK>
97 void unpack_recv_buffer(const DATA_TYPE* recv_buffer, int recv_buffer_size,
98 array::ArrayView<DATA_TYPE, RANK>& array,
99 const bool on_device) const;
100
101 template <int ParallelDim, typename DATA_TYPE, int RANK>
102 void pack_recv_adjoint_buffer(const array::ArrayView<DATA_TYPE, RANK>& array,
103 DATA_TYPE* recv_buffer, int recv_buffer_size,
104 const bool on_device) const;
105
106 template <int ParallelDim, typename DATA_TYPE, int RANK>
107 void unpack_send_adjoint_buffer(const DATA_TYPE* send_buffer, int send_buffer_size,
108 array::ArrayView<DATA_TYPE, RANK>& array,
109 const bool on_device) const;
110
111 template <int ParallelDim, typename DATA_TYPE, int RANK>
112 void zero_halos(array::ArrayView<DATA_TYPE, RANK>& array,
113 DATA_TYPE* recv_buffer, int recv_buffer_size,
114 const bool on_device) const;
115
116 template <typename DATA_TYPE, int RANK>
117 void var_info(const array::ArrayView<DATA_TYPE, RANK>& arr, std::vector<idx_t>& varstrides,
118 std::vector<idx_t>& varshape) const;
119
120 const mpi::Comm& comm() const {
121 return *comm_;
122 }
123
124 int devices() const;
125 bool is_device_accessible(const void* ptr) const;
126
127 private: // data
128 std::string name_;
129 bool is_setup_;
130
131 int sendcnt_;
132 int recvcnt_;
133 std::vector<int> sendcounts_;
134 std::vector<int> senddispls_;
135 std::vector<int> recvcounts_;
136 std::vector<int> recvdispls_;
137 array::SVector<int> sendmap_;
138 array::SVector<int> recvmap_;
139 int parsize_;
140
141 int nproc;
142 int myproc;
143 const mpi::Comm* comm_;
144
145 public:
146 struct Backdoor {
147 int parsize;
148 } backdoor;
149 };
150
151 template <typename DATA_TYPE, int RANK, typename ParallelDim>
152 void HaloExchange::execute(array::Array& field, bool on_device) const {
153 ATLAS_TRACE(std::string("HaloExchange::execute")+(on_device?"[device]":"[host]"), {std::string("halo-exchange"), std::string("halo-exchange")+(on_device?".device":".host")});
154 if (!is_setup_) {
155 throw_Exception("HaloExchange was not setup", Here());
156 }
157
158 on_device = on_device && devices() > 0;
159
160 bool update_device_at_end = false;
161 if (on_device) {
162 ATLAS_ASSERT(field.deviceNeedsUpdate() == false);
163
164 if (not ATLAS_HAVE_GPU_AWARE_MPI) {
165 // Without GPU aware MPI, we need to do halo exchange on host and copy back to device
166 update_device_at_end = on_device;
167 on_device = false;
168 field.syncHost();
169 }
170 else {
171 field.syncDevice();
172 }
173 }
174 else {
175 field.syncHost();
176 }
177
178 auto array_view =
179 on_device ? array::make_device_view<DATA_TYPE, RANK>(field) : array::make_host_view<DATA_TYPE, RANK>(field);
180
181 constexpr int parallelDim = array::get_parallel_dim<ParallelDim>(array_view);
182 idx_t var_size = array::get_var_size<parallelDim>(array_view);
183
184 int tag(1);
185 std::size_t nproc_loc(static_cast<std::size_t>(nproc));
186 std::vector<int> inner_counts(nproc_loc), halo_counts(nproc_loc);
187 std::vector<int> inner_counts_init(nproc_loc), halo_counts_init(nproc_loc);
188 std::vector<int> inner_displs(nproc_loc), halo_displs(nproc_loc);
189 std::vector<eckit::mpi::Request> inner_req(nproc_loc), halo_req(nproc_loc);
190
191 int inner_size = sendcnt_ * var_size;
192 int halo_size = recvcnt_ * var_size;
193 DATA_TYPE* inner_buffer = allocate_buffer<DATA_TYPE>(inner_size, on_device);
194 DATA_TYPE* halo_buffer = allocate_buffer<DATA_TYPE>(halo_size, on_device);
195
196 if (on_device) {
197 ATLAS_ASSERT( is_device_accessible(inner_buffer) );
198 ATLAS_ASSERT( is_device_accessible(halo_buffer) );
199 ATLAS_ASSERT( is_device_accessible(array_view.data()) );
200 }
201
202 counts_displs_setup<DATA_TYPE>(var_size, inner_counts_init, halo_counts_init, inner_counts, halo_counts,
203 inner_displs, halo_displs);
204
205 ireceive<DATA_TYPE>(tag, halo_displs, halo_counts, halo_req, halo_buffer);
206
207 /// Pack
208 pack_send_buffer<parallelDim>(array_view, inner_buffer, inner_size, on_device);
209
210 isend_and_wait_for_receive<DATA_TYPE>(tag, halo_counts_init, halo_req, inner_displs, inner_counts, inner_req,
211 inner_buffer);
212
213 /// Unpack
214 unpack_recv_buffer<parallelDim>(halo_buffer, halo_size, array_view, on_device);
215
216 wait_for_send(inner_counts_init, inner_req);
217
218 deallocate_buffer<DATA_TYPE>(inner_buffer, inner_size, on_device);
219 deallocate_buffer<DATA_TYPE>(halo_buffer, halo_size, on_device);
220
221 on_device ? field.setHostNeedsUpdate(true) : field.setDeviceNeedsUpdate(true);
222 if (update_device_at_end) {
223 field.syncDevice();
224 }
225 }
226
227 template <typename DATA_TYPE, int RANK, typename ParallelDim>
228 void HaloExchange::execute_adjoint(array::Array& field, bool on_device) const {
229 if (!is_setup_) {
230 throw_Exception("HaloExchange was not setup", Here());
231 }
232 ATLAS_TRACE(std::string("HaloExchange::execute_adjoint")+(on_device?"[device]":"[host]"), {std::string("halo-exchange-adjoint"),std::string("halo-exchange-adjoint")+(on_device?".device":".host")});
233
234 on_device = on_device && devices() > 0;
235
236 // adjoint on device is not yet implemented!
237 // So copy to host and do halo-exchange there, and copy back to device
238 bool update_device_at_end = on_device;
239 if (on_device) {
240 ATLAS_ASSERT(field.deviceNeedsUpdate() == false);
241 field.syncHost();
242 on_device = false;
243 }
244
245 auto array_view =
246 on_device ? array::make_device_view<DATA_TYPE, RANK>(field) : array::make_host_view<DATA_TYPE, RANK>(field);
247
248 constexpr int parallelDim = array::get_parallel_dim<ParallelDim>(array_view);
249 idx_t var_size = array::get_var_size<parallelDim>(array_view);
250
251 int tag(1);
252 std::size_t nproc_loc(static_cast<std::size_t>(nproc));
253 std::vector<int> halo_counts(nproc_loc), inner_counts(nproc_loc);
254 std::vector<int> halo_counts_init(nproc_loc), inner_counts_init(nproc_loc);
255 std::vector<int> halo_displs(nproc_loc), inner_displs(nproc_loc);
256 std::vector<eckit::mpi::Request> halo_req(nproc_loc), inner_req(nproc_loc);
257
258 int halo_size = sendcnt_ * var_size;
259 int inner_size = recvcnt_ * var_size;
260 DATA_TYPE* halo_buffer = allocate_buffer<DATA_TYPE>(halo_size, on_device);
261 DATA_TYPE* inner_buffer = allocate_buffer<DATA_TYPE>(inner_size, on_device);
262
263 counts_displs_setup<DATA_TYPE>(var_size, halo_counts_init, inner_counts_init, halo_counts, inner_counts,
264 halo_displs, inner_displs);
265
266 ireceive<DATA_TYPE>(tag, halo_displs, halo_counts, halo_req, halo_buffer);
267
268 /// Pack
269 pack_recv_adjoint_buffer<parallelDim>(array_view, inner_buffer, inner_size, on_device);
270
271 /// Send
272 isend_and_wait_for_receive<DATA_TYPE>(tag, halo_counts_init, halo_req, inner_displs, inner_counts, inner_req,
273 inner_buffer);
274
275 /// Unpack
276 unpack_send_adjoint_buffer<parallelDim>(halo_buffer, halo_size, array_view, on_device);
277
278 /// Wait for sending to finish
279 wait_for_send(inner_counts_init, inner_req);
280
281 zero_halos<parallelDim>(array_view, halo_buffer, halo_size, on_device);
282
283 deallocate_buffer<DATA_TYPE>(halo_buffer, halo_size, on_device);
284 deallocate_buffer<DATA_TYPE>(inner_buffer, inner_size, on_device);
285
286 on_device ? field.setHostNeedsUpdate(true) : field.setDeviceNeedsUpdate(true);
287 if (update_device_at_end) {
288 field.syncDevice();
289 }
290 }
291
292 template <typename DATA_TYPE>
293 DATA_TYPE* HaloExchange::allocate_buffer(const int buffer_size, const bool on_device) const {
294 DATA_TYPE* buffer{nullptr};
295
296 if (on_device) {
297 util::allocate_devicemem(buffer, buffer_size);
298 }
299 else {
300 util::allocate_hostmem(buffer, buffer_size);
301 }
302
303 return buffer;
304 }
305
306
307 template <typename DATA_TYPE>
308 void HaloExchange::deallocate_buffer(DATA_TYPE* buffer, const int buffer_size, const bool on_device) const {
309 if (on_device) {
310 util::delete_devicemem(buffer, buffer_size);
311 }
312 else {
313 util::delete_hostmem(buffer, buffer_size);
314 }
315 }
316
317
318 template <typename DATA_TYPE>
319 void HaloExchange::counts_displs_setup(const idx_t var_size, std::vector<int>& send_counts_init,
320 std::vector<int>& recv_counts_init, std::vector<int>& send_counts,
321 std::vector<int>& recv_counts, std::vector<int>& send_displs,
322 std::vector<int>& recv_displs) const {
323 for (size_t jproc = 0; jproc < static_cast<size_t>(nproc); ++jproc) {
324 send_counts_init[jproc] = sendcounts_[jproc];
325 recv_counts_init[jproc] = recvcounts_[jproc];
326 send_counts[jproc] = sendcounts_[jproc] * var_size;
327 recv_counts[jproc] = recvcounts_[jproc] * var_size;
328 send_displs[jproc] = senddispls_[jproc] * var_size;
329 recv_displs[jproc] = recvdispls_[jproc] * var_size;
330 }
331 }
332
333 template <typename DATA_TYPE>
334 void HaloExchange::ireceive(int tag, std::vector<int>& recv_displs, std::vector<int>& recv_counts,
335 std::vector<eckit::mpi::Request>& recv_req, DATA_TYPE* recv_buffer) const {
336 ATLAS_TRACE_MPI(IRECEIVE) {
337 /// Let MPI know what we like to receive
338 for (size_t jproc = 0; jproc < static_cast<size_t>(nproc); ++jproc) {
339 if (recv_counts[jproc] > 0) {
340 recv_req[jproc] =
341 comm().iReceive(recv_buffer+recv_displs[jproc], recv_counts[jproc], jproc, tag);
342 }
343 }
344 }
345 }
346
347 template <typename DATA_TYPE>
348 void HaloExchange::isend_and_wait_for_receive(int tag, std::vector<int>& recv_counts_init,
349 std::vector<eckit::mpi::Request>& recv_req, std::vector<int>& send_displs,
350 std::vector<int>& send_counts, std::vector<eckit::mpi::Request>& send_req,
351 DATA_TYPE* send_buffer) const {
352 /// Send
353 ATLAS_TRACE_MPI(ISEND) {
354 for (size_t jproc = 0; jproc < static_cast<size_t>(nproc); ++jproc) {
355 if (send_counts[jproc] > 0) {
356 send_req[jproc] = comm().iSend(send_buffer+send_displs[jproc], send_counts[jproc], jproc, tag);
357 }
358 }
359 }
360
361 /// Wait for receiving to finish
362 ATLAS_TRACE_MPI(WAIT, "mpi-wait receive") {
363 for (size_t jproc = 0; jproc < static_cast<size_t>(nproc); ++jproc) {
364 if (recv_counts_init[jproc] > 0) {
365 comm().wait(recv_req[jproc]);
366 }
367 }
368 }
369 }
370
371 template <int ParallelDim, typename DATA_TYPE, int RANK>
372 void HaloExchange::zero_halos(array::ArrayView<DATA_TYPE, RANK>& array,
373 DATA_TYPE* recv_buffer, int recv_size,
374 ATLAS_MAYBE_UNUSED const bool on_device) const {
375 ATLAS_TRACE();
376 detail::Zeroer<ParallelDim, DATA_TYPE, RANK>::zero(recvcnt_, recvmap_, array, recv_buffer, recv_size, on_device);
377 }
378
379 template <int ParallelDim, typename DATA_TYPE, int RANK>
380 void HaloExchange::pack_send_buffer(const array::ArrayView<DATA_TYPE, RANK>& array,
381 DATA_TYPE* send_buffer, int send_size,
382 const bool on_device) const {
383 ATLAS_TRACE();
384 detail::Packer<ParallelDim, DATA_TYPE, RANK>::pack(sendcnt_, sendmap_, array, send_buffer,
385 send_size, on_device);
386 }
387
388 template <int ParallelDim, typename DATA_TYPE, int RANK>
389 void HaloExchange::unpack_recv_buffer(const DATA_TYPE* recv_buffer, int recv_size,
390 array::ArrayView<DATA_TYPE, RANK>& array,
391 const bool on_device) const {
392 ATLAS_TRACE();
393 detail::Packer<ParallelDim, DATA_TYPE, RANK>::unpack(recvcnt_, recvmap_, recv_buffer, recv_size, array, on_device);
394 }
395
396 template <int ParallelDim, typename DATA_TYPE, int RANK>
397 void HaloExchange::pack_recv_adjoint_buffer(const array::ArrayView<DATA_TYPE, RANK>& array,
398 DATA_TYPE* recv_buffer, int recv_size, const bool on_device) const {
399 ATLAS_TRACE();
400 detail::AdjointPacker<ParallelDim, DATA_TYPE, RANK>::pack(recvcnt_, recvmap_, array, recv_buffer, recv_size, on_device);
401 }
402
403 template <int ParallelDim, typename DATA_TYPE, int RANK>
404 void HaloExchange::unpack_send_adjoint_buffer(const DATA_TYPE* send_buffer, int send_size,
405 array::ArrayView<DATA_TYPE, RANK>& array,
406 ATLAS_MAYBE_UNUSED const bool on_device) const {
407 ATLAS_TRACE();
408 detail::AdjointPacker<ParallelDim, DATA_TYPE, RANK>::unpack(sendcnt_, sendmap_, send_buffer, send_size, array, on_device);
409 }
410
411 // template<typename DATA_TYPE>
412 // void HaloExchange::execute( DATA_TYPE field[], idx_t nb_vars ) const
413 //{
414 // throw_AssertionFailed("Call not supported");
415
416 // idx_t strides[] = {1};
417 // idx_t shape[] = {nb_vars};
418 // execute( field, strides, shape, 1);
419 //}
420
421 // template <typename DATA_TYPE, int RANK>
422 // void HaloExchange::execute( array::ArrayView<DATA_TYPE,RANK>&& field ) const
423 //{
424 // execute(field);
425 //}
426
427 //----------------------------------------------------------------------------------------------------------------------
428 // C wrapper interfaces to C++ routines
429 extern "C" {
430 HaloExchange* atlas__HaloExchange__new();
431 void atlas__HaloExchange__delete(HaloExchange* This);
432 void atlas__HaloExchange__setup(HaloExchange* This, int part[], idx_t remote_idx[], int base, int size);
433 void atlas__HaloExchange__execute_strided_int(HaloExchange* This, int field[], int var_strides[], int var_shape[],
434 int var_rank);
435 void atlas__HaloExchange__execute_strided_long(HaloExchange* This, long field[], int var_strides[], int var_shape[],
436 int var_rank);
437 void atlas__HaloExchange__execute_strided_float(HaloExchange* This, float field[], int var_strides[], int var_shape[],
438 int var_rank);
439 void atlas__HaloExchange__execute_strided_double(HaloExchange* This, double field[], int var_strides[], int var_shape[],
440 int var_rank);
441 void atlas__HaloExchange__execute_int(HaloExchange* This, int field[], int var_rank);
442 void atlas__HaloExchange__execute_float(HaloExchange* This, float field[], int var_rank);
443 void atlas__HaloExchange__execute_double(HaloExchange* This, double field[], int var_rank);
444
445 void atlas__HaloExchange__execute_adjoint_strided_int(HaloExchange* This, int field[], int var_strides[],
446 int var_shape[], int var_rank);
447 void atlas__HaloExchange__execute_adjoint_strided_long(HaloExchange* This, long field[], int var_strides[],
448 int var_shape[], int var_rank);
449 void atlas__HaloExchange__execute_adjoint_strided_float(HaloExchange* This, float field[], int var_strides[],
450 int var_shape[], int var_rank);
451 void atlas__HaloExchange__execute_adjoint_strided_double(HaloExchange* This, double field[], int var_strides[],
452 int var_shape[], int var_rank);
453 void atlas__HaloExchange__execute_adjoint_int(HaloExchange* This, int field[], int var_rank);
454 void atlas__HaloExchange__execute_adjoint_float(HaloExchange* This, float field[], int var_rank);
455 void atlas__HaloExchange__execute_adjoint_double(HaloExchange* This, double field[], int var_rank);
456 }
457
458 //----------------------------------------------------------------------------------------------------------------------
459
460 } // namespace parallel
461 } // namespace atlas