/* * (C) Copyright 2025- ECMWF. * * This software is licensed under the terms of the Apache Licence Version 2.0 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. * In applying this licence, ECMWF does not waive the privileges and immunities * granted to it by virtue of its status as an intergovernmental organisation * nor does it submit to any jurisdiction. */ #include #include "atlas/array/Array.h" #include "atlas/array/ArrayViewUtil.h" #include "atlas/parallel/mpi/mpi.h" #include "atlas/parallel/detail/Packer.h" #include "atlas/runtime/Log.h" #include "atlas/runtime/Trace.h" #include "atlas/runtime/Exception.h" namespace atlas::parallel { class Collect { public: void setup(const idx_t recv_size, const int recv_part[], const idx_t recv_remote_idx[], const int recv_idx_base); void setup(const std::string& mpi_comm, const idx_t recv_size, const int recv_part[], const idx_t recv_remote_idx[], const int recv_idx_base); template void execute(array::Array& send, array::Array& recv, bool on_device = false) const; private: template DATA_TYPE* allocate_buffer(const int buffer_size, const bool on_device) const; template void deallocate_buffer(DATA_TYPE* buffer, const int buffer_size, const bool on_device) const; void counts_displs_setup(const idx_t var_size, std::vector& send_counts_init, std::vector& recv_counts_init, std::vector& send_counts, std::vector& recv_counts, std::vector& send_displs, std::vector& recv_displs) const; template void ireceive(int tag, std::vector& recv_displs, std::vector& recv_counts, std::vector& recv_req, DATA_TYPE* recv_buffer) const; template void pack_send_buffer(const array::ArrayView& array, DATA_TYPE* send_buffer, int send_size, const bool on_device) const; template void unpack_recv_buffer(const DATA_TYPE* recv_buffer, int recv_size, array::ArrayView& array, const bool on_device) const; template void isend_and_wait_for_receive(int tag, std::vector& recv_counts_init, std::vector& recv_req, std::vector& send_displs, std::vector& send_counts, std::vector& send_req, DATA_TYPE* send_buffer) const; void wait_for_send(std::vector& send_counts_init, std::vector& send_req) const; const mpi::Comm& comm() const { return *comm_; } int devices() const; private: bool is_setup_; int sendcnt_; int recvcnt_; std::vector sendcounts_; std::vector senddispls_; std::vector recvcounts_; std::vector recvdispls_; array::SVector sendmap_; array::SVector recvmap_; int mpi_size_; int mpi_rank_; const mpi::Comm* comm_; }; template void Collect::execute(array::Array& send, array::Array& recv, bool on_device) const { ATLAS_TRACE("Collect", {"collect"}); if (!is_setup_) { throw_Exception("Collect was not setup", Here()); } if( send.rank() != recv.rank() ){ throw_Exception("send and recv arrays are of different rank"); } on_device = on_device && devices() > 0; auto send_view = on_device ? array::make_device_view(send) : array::make_host_view(send); auto recv_view = on_device ? array::make_device_view(recv) : array::make_host_view(recv); constexpr int parallelDim = array::get_parallel_dim(send_view); idx_t var_size = array::get_var_size(send_view); int tag(1); std::size_t nproc_loc(static_cast(mpi_size_)); std::vector send_counts(nproc_loc), recv_counts(nproc_loc); std::vector send_counts_init(nproc_loc), recv_counts_init(nproc_loc); std::vector send_displs(nproc_loc), recv_displs(nproc_loc); std::vector send_req(nproc_loc), recv_req(nproc_loc); int send_size = sendcnt_ * var_size; int recv_size = recvcnt_ * var_size; DATA_TYPE* send_buffer = allocate_buffer(send_size, on_device); DATA_TYPE* recv_buffer = allocate_buffer(recv_size, on_device); counts_displs_setup(var_size, send_counts_init, recv_counts_init, send_counts, recv_counts, send_displs, recv_displs); ireceive(tag, recv_displs, recv_counts, recv_req, recv_buffer); /// Pack pack_send_buffer(send_view, send_buffer, send_size, on_device); isend_and_wait_for_receive(tag, recv_counts_init, recv_req, send_displs, send_counts, send_req, send_buffer); /// Unpack unpack_recv_buffer(recv_buffer, recv_size, recv_view, on_device); wait_for_send(send_counts_init, send_req); deallocate_buffer(send_buffer, send_size, on_device); deallocate_buffer(recv_buffer, recv_size, on_device); } template DATA_TYPE* Collect::allocate_buffer(const int buffer_size, const bool on_device) const { DATA_TYPE* buffer{nullptr}; if (on_device) { util::allocate_devicemem(buffer, buffer_size); } else { util::allocate_hostmem(buffer, buffer_size); } return buffer; } template void Collect::deallocate_buffer(DATA_TYPE* buffer, const int buffer_size, const bool on_device) const { if (on_device) { util::delete_devicemem(buffer, buffer_size); } else { util::delete_hostmem(buffer, buffer_size); } } template void Collect::ireceive(int tag, std::vector& recv_displs, std::vector& recv_counts, std::vector& recv_req, DATA_TYPE* recv_buffer) const { ATLAS_TRACE_MPI(IRECEIVE) { /// Let MPI know what we like to receive for (size_t jproc = 0; jproc < static_cast(mpi_size_); ++jproc) { if (recv_counts[jproc] > 0) { recv_req[jproc] = comm().iReceive(recv_buffer+recv_displs[jproc], recv_counts[jproc], jproc, tag); } } } } template void Collect::pack_send_buffer(const array::ArrayView& array, DATA_TYPE* send_buffer, int send_size, const bool on_device) const { ATLAS_TRACE(); detail::Packer::pack(sendcnt_, sendmap_, array, send_buffer, send_size, on_device); } template void Collect::unpack_recv_buffer(const DATA_TYPE* recv_buffer, int recv_size, array::ArrayView& array, const bool on_device) const { ATLAS_TRACE(); detail::Packer::unpack(recvcnt_, recvmap_, recv_buffer, recv_size, array, on_device); } template void Collect::isend_and_wait_for_receive(int tag, std::vector& recv_counts_init, std::vector& recv_req, std::vector& send_displs, std::vector& send_counts, std::vector& send_req, DATA_TYPE* send_buffer) const { /// Send ATLAS_TRACE_MPI(ISEND) { for (size_t jproc = 0; jproc < static_cast(mpi_size_); ++jproc) { if (send_counts[jproc] > 0) { send_req[jproc] = comm().iSend(send_buffer+send_displs[jproc], send_counts[jproc], jproc, tag); } } } /// Wait for receiving to finish ATLAS_TRACE_MPI(WAIT, "mpi-wait receive") { for (size_t jproc = 0; jproc < static_cast(mpi_size_); ++jproc) { if (recv_counts_init[jproc] > 0) { comm().wait(recv_req[jproc]); } } } } } // namespace atlas::parallel