| 1 |
/* |
| 2 |
* (C) Copyright 2025 ECMWF. |
| 3 |
* |
| 4 |
* This software is licensed under the terms of the Apache Licence Version 2.0 |
| 5 |
* which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. |
| 6 |
* In applying this licence, ECMWF does not waive the privileges and immunities |
| 7 |
* granted to it by virtue of its status as an intergovernmental organisation |
| 8 |
* nor does it submit to any jurisdiction. |
| 9 |
*/ |
| 10 |
|
| 11 |
#include "atlas/functionspace/Locate.h" |
| 12 |
|
| 13 |
#include "atlas/functionspace/FunctionSpace.h" |
| 14 |
#include "atlas/array/ArrayView.h" |
| 15 |
#include "atlas/array/Array.h" |
| 16 |
#include "atlas/option.h" |
| 17 |
#include "atlas/parallel/mpi/mpi.h" |
| 18 |
|
| 19 |
namespace atlas::functionspace { |
| 20 |
|
| 21 |
// Figure out the global distribution using the functionspace partition fields from each rank. |
| 22 |
static atlas::vector<int> allgather_distribution_array(const FunctionSpace& fs) { |
| 23 |
auto& comm = atlas::mpi::comm(fs.mpi_comm()); |
| 24 |
int mpi_rank = comm.rank(); |
| 25 |
int mpi_root = 0; |
| 26 |
|
| 27 |
atlas::vector<int> distribution_array; |
| 28 |
ATLAS_TRACE_SCOPE("Gather global partition array") { |
| 29 |
Field field_partition_global = fs.createField(fs.partition(), option::global(mpi_root)); |
| 30 |
fs.gather(fs.partition(), field_partition_global); |
| 31 |
std::size_t partition_global_size = field_partition_global.size(); |
| 32 |
comm.broadcast(partition_global_size, mpi_root); |
| 33 |
distribution_array.resize(partition_global_size); |
| 34 |
if (mpi_rank == mpi_root) { |
| 35 |
int* data = field_partition_global.array().host_data<int>(); |
| 36 |
distribution_array.assign(data, data+partition_global_size); |
| 37 |
} |
| 38 |
comm.broadcast(distribution_array.data(), distribution_array.size(), mpi_root); |
| 39 |
} |
| 40 |
return distribution_array; |
| 41 |
} |
| 42 |
|
| 43 |
Locator::Locator(const FunctionSpace& fs) : |
| 44 |
fs_global_index_{fs.global_index()}, |
| 45 |
fs_ghost_{fs.ghost()} { |
| 46 |
distribution_array_ = allgather_distribution_array(fs); |
| 47 |
distribution_function_ = [this](gidx_t j){ return distribution_array_[j]; }; |
| 48 |
distribution_size_ = distribution_array_.size(); |
| 49 |
mpi_comm_ = fs.mpi_comm(); |
| 50 |
} |
| 51 |
|
| 52 |
void Locator::locate( |
| 53 |
// input |
| 54 |
span<const gidx_t> global_index, const gidx_t global_index_base, |
| 55 |
// output |
| 56 |
span<int> partition, const int partition_base, |
| 57 |
span<idx_t> remote_index, const idx_t remote_index_base) const { |
| 58 |
constexpr gidx_t fs_global_index_base_ = 1; |
| 59 |
constexpr int distribution_base_ = 0; |
| 60 |
if (distribution_array_.size()) { |
| 61 |
::atlas::parallel::Locator::locate( |
| 62 |
// context |
| 63 |
mpi_comm_, |
| 64 |
array::make_view<const gidx_t,1>(fs_global_index_).as_mdspan(), fs_global_index_base_, |
| 65 |
array::make_view<const int,1>(fs_ghost_).as_mdspan(), |
| 66 |
span<const int>{distribution_array_.data(), distribution_array_.size()}, distribution_base_, |
| 67 |
// input |
| 68 |
global_index, global_index_base, |
| 69 |
// output |
| 70 |
partition, partition_base, |
| 71 |
remote_index, remote_index_base); |
| 72 |
} |
| 73 |
else { |
| 74 |
::atlas::parallel::Locator::locate( |
| 75 |
// context |
| 76 |
mpi_comm_, |
| 77 |
array::make_view<const gidx_t,1>(fs_global_index_).as_mdspan(), |
| 78 |
fs_global_index_base_, |
| 79 |
array::make_view<const int,1>(fs_ghost_).as_mdspan(), |
| 80 |
fspan<const int>{&distribution_function_, distribution_size_}, distribution_base_, |
| 81 |
// input |
| 82 |
global_index, global_index_base, |
| 83 |
// output |
| 84 |
partition, partition_base, |
| 85 |
remote_index, remote_index_base); |
| 86 |
} |
| 87 |
} |
| 88 |
|
| 89 |
} // namespace atlas::functionspace |