Debian Codegraph @

src/atlas/parallel/Collect.h

raw
1 /*
2 * (C) Copyright 2025- ECMWF.
3 *
4 * This software is licensed under the terms of the Apache Licence Version 2.0
5 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
6 * In applying this licence, ECMWF does not waive the privileges and immunities
7 * granted to it by virtue of its status as an intergovernmental organisation
8 * nor does it submit to any jurisdiction.
9 */
10
11 #include <vector>
12
13 #include "atlas/array/Array.h"
14 #include "atlas/array/ArrayViewUtil.h"
15 #include "atlas/parallel/mpi/mpi.h"
16 #include "atlas/parallel/detail/Packer.h"
17
18 #include "atlas/runtime/Log.h"
19 #include "atlas/runtime/Trace.h"
20 #include "atlas/runtime/Exception.h"
21
22 namespace atlas::parallel {
23
24 class Collect {
25 public:
26 void setup(const idx_t recv_size, const int recv_part[], const idx_t recv_remote_idx[], const int recv_idx_base);
27
28 void setup(const std::string& mpi_comm,
29 const idx_t recv_size, const int recv_part[], const idx_t recv_remote_idx[], const int recv_idx_base);
30
31 template <typename DATA_TYPE, int RANK, typename ParallelDim = array::FirstDim>
32 void execute(array::Array& send, array::Array& recv, bool on_device = false) const;
33
34 private:
35
36 template <typename DATA_TYPE>
37 DATA_TYPE* allocate_buffer(const int buffer_size, const bool on_device) const;
38
39 template <typename DATA_TYPE>
40 void deallocate_buffer(DATA_TYPE* buffer, const int buffer_size, const bool on_device) const;
41
42 void counts_displs_setup(const idx_t var_size, std::vector<int>& send_counts_init,
43 std::vector<int>& recv_counts_init, std::vector<int>& send_counts,
44 std::vector<int>& recv_counts, std::vector<int>& send_displs,
45 std::vector<int>& recv_displs) const;
46
47 template <typename DATA_TYPE>
48 void ireceive(int tag, std::vector<int>& recv_displs, std::vector<int>& recv_counts,
49 std::vector<eckit::mpi::Request>& recv_req, DATA_TYPE* recv_buffer) const;
50
51 template <int ParallelDim, typename DATA_TYPE, int RANK>
52 void pack_send_buffer(const array::ArrayView<DATA_TYPE, RANK>& array,
53 DATA_TYPE* send_buffer, int send_size,
54 const bool on_device) const;
55
56 template <int ParallelDim, typename DATA_TYPE, int RANK>
57 void unpack_recv_buffer(const DATA_TYPE* recv_buffer, int recv_size,
58 array::ArrayView<DATA_TYPE, RANK>& array,
59 const bool on_device) const;
60
61 template <typename DATA_TYPE>
62 void isend_and_wait_for_receive(int tag, std::vector<int>& recv_counts_init,
63 std::vector<eckit::mpi::Request>& recv_req, std::vector<int>& send_displs,
64 std::vector<int>& send_counts, std::vector<eckit::mpi::Request>& send_req,
65 DATA_TYPE* send_buffer) const;
66
67 void wait_for_send(std::vector<int>& send_counts_init, std::vector<eckit::mpi::Request>& send_req) const;
68
69 const mpi::Comm& comm() const {
70 return *comm_;
71 }
72
73 int devices() const;
74
75 private:
76 bool is_setup_;
77
78 int sendcnt_;
79 int recvcnt_;
80 std::vector<int> sendcounts_;
81 std::vector<int> senddispls_;
82 std::vector<int> recvcounts_;
83 std::vector<int> recvdispls_;
84 array::SVector<int> sendmap_;
85 array::SVector<int> recvmap_;
86
87 int mpi_size_;
88 int mpi_rank_;
89 const mpi::Comm* comm_;
90 };
91
92 template <typename DATA_TYPE, int RANK, typename ParallelDim>
93 void Collect::execute(array::Array& send, array::Array& recv, bool on_device) const {
94 ATLAS_TRACE("Collect", {"collect"});
95 if (!is_setup_) {
96 throw_Exception("Collect was not setup", Here());
97 }
98
99 if( send.rank() != recv.rank() ){
100 throw_Exception("send and recv arrays are of different rank");
101 }
102
103 on_device = on_device && devices() > 0;
104
105 auto send_view =
106 on_device ? array::make_device_view<DATA_TYPE, RANK>(send) : array::make_host_view<DATA_TYPE, RANK>(send);
107 auto recv_view =
108 on_device ? array::make_device_view<DATA_TYPE, RANK>(recv) : array::make_host_view<DATA_TYPE, RANK>(recv);
109
110 constexpr int parallelDim = array::get_parallel_dim<ParallelDim>(send_view);
111 idx_t var_size = array::get_var_size<parallelDim>(send_view);
112
113 int tag(1);
114 std::size_t nproc_loc(static_cast<std::size_t>(mpi_size_));
115 std::vector<int> send_counts(nproc_loc), recv_counts(nproc_loc);
116 std::vector<int> send_counts_init(nproc_loc), recv_counts_init(nproc_loc);
117 std::vector<int> send_displs(nproc_loc), recv_displs(nproc_loc);
118 std::vector<eckit::mpi::Request> send_req(nproc_loc), recv_req(nproc_loc);
119
120 int send_size = sendcnt_ * var_size;
121 int recv_size = recvcnt_ * var_size;
122 DATA_TYPE* send_buffer = allocate_buffer<DATA_TYPE>(send_size, on_device);
123 DATA_TYPE* recv_buffer = allocate_buffer<DATA_TYPE>(recv_size, on_device);
124
125 counts_displs_setup(var_size, send_counts_init, recv_counts_init, send_counts, recv_counts,
126 send_displs, recv_displs);
127
128 ireceive<DATA_TYPE>(tag, recv_displs, recv_counts, recv_req, recv_buffer);
129
130 /// Pack
131 pack_send_buffer<parallelDim>(send_view, send_buffer, send_size, on_device);
132
133 isend_and_wait_for_receive<DATA_TYPE>(tag, recv_counts_init, recv_req, send_displs, send_counts, send_req,
134 send_buffer);
135
136 /// Unpack
137 unpack_recv_buffer<parallelDim>(recv_buffer, recv_size, recv_view, on_device);
138
139 wait_for_send(send_counts_init, send_req);
140
141 deallocate_buffer<DATA_TYPE>(send_buffer, send_size, on_device);
142 deallocate_buffer<DATA_TYPE>(recv_buffer, recv_size, on_device);
143
144 }
145
146 template <typename DATA_TYPE>
147 DATA_TYPE* Collect::allocate_buffer(const int buffer_size, const bool on_device) const {
148 DATA_TYPE* buffer{nullptr};
149
150 if (on_device) {
151 util::allocate_devicemem(buffer, buffer_size);
152 }
153 else {
154 util::allocate_hostmem(buffer, buffer_size);
155 }
156
157 return buffer;
158 }
159
160 template <typename DATA_TYPE>
161 void Collect::deallocate_buffer(DATA_TYPE* buffer, const int buffer_size, const bool on_device) const {
162 if (on_device) {
163 util::delete_devicemem(buffer, buffer_size);
164 }
165 else {
166 util::delete_hostmem(buffer, buffer_size);
167 }
168 }
169
170 template <typename DATA_TYPE>
171 void Collect::ireceive(int tag, std::vector<int>& recv_displs, std::vector<int>& recv_counts,
172 std::vector<eckit::mpi::Request>& recv_req, DATA_TYPE* recv_buffer) const {
173 ATLAS_TRACE_MPI(IRECEIVE) {
174 /// Let MPI know what we like to receive
175 for (size_t jproc = 0; jproc < static_cast<size_t>(mpi_size_); ++jproc) {
176 if (recv_counts[jproc] > 0) {
177 recv_req[jproc] =
178 comm().iReceive(recv_buffer+recv_displs[jproc], recv_counts[jproc], jproc, tag);
179 }
180 }
181 }
182 }
183
184 template <int ParallelDim, typename DATA_TYPE, int RANK>
185 void Collect::pack_send_buffer(const array::ArrayView<DATA_TYPE, RANK>& array,
186 DATA_TYPE* send_buffer, int send_size,
187 const bool on_device) const {
188 ATLAS_TRACE();
189 detail::Packer<ParallelDim, DATA_TYPE, RANK>::pack(sendcnt_, sendmap_, array, send_buffer, send_size, on_device);
190 }
191
192 template <int ParallelDim, typename DATA_TYPE, int RANK>
193 void Collect::unpack_recv_buffer(const DATA_TYPE* recv_buffer, int recv_size,
194 array::ArrayView<DATA_TYPE, RANK>& array,
195 const bool on_device) const {
196 ATLAS_TRACE();
197 detail::Packer<ParallelDim, DATA_TYPE, RANK>::unpack(recvcnt_, recvmap_, recv_buffer, recv_size, array, on_device);
198 }
199
200 template <typename DATA_TYPE>
201 void Collect::isend_and_wait_for_receive(int tag, std::vector<int>& recv_counts_init,
202 std::vector<eckit::mpi::Request>& recv_req, std::vector<int>& send_displs,
203 std::vector<int>& send_counts, std::vector<eckit::mpi::Request>& send_req,
204 DATA_TYPE* send_buffer) const {
205 /// Send
206 ATLAS_TRACE_MPI(ISEND) {
207 for (size_t jproc = 0; jproc < static_cast<size_t>(mpi_size_); ++jproc) {
208 if (send_counts[jproc] > 0) {
209 send_req[jproc] = comm().iSend(send_buffer+send_displs[jproc], send_counts[jproc], jproc, tag);
210 }
211 }
212 }
213
214 /// Wait for receiving to finish
215 ATLAS_TRACE_MPI(WAIT, "mpi-wait receive") {
216 for (size_t jproc = 0; jproc < static_cast<size_t>(mpi_size_); ++jproc) {
217 if (recv_counts_init[jproc] > 0) {
218 comm().wait(recv_req[jproc]);
219 }
220 }
221 }
222 }
223
224 } // namespace atlas::parallel