Debian Codegraph @

src/atlas/parallel/Locate.cc

raw
1 /*
2 * (C) Copyright 2025 ECMWF.
3 *
4 * This software is licensed under the terms of the Apache Licence Version 2.0
5 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
6 * In applying this licence, ECMWF does not waive the privileges and immunities
7 * granted to it by virtue of its status as an intergovernmental organisation
8 * nor does it submit to any jurisdiction.
9 */
10
11 #include "Locate.h"
12
13 #include <unordered_map>
14
15 #include "atlas/parallel/mpi/mpi.h"
16 #include "atlas/runtime/Trace.h"
17
18 namespace atlas::parallel {
19
20 // Given distribution and global_index, find the corresponding partition for each global index
21 // The global-index is typically 1-based
22 void Locator::locate_partition(
23 fspan<const int> distribution, const int distribution_base,
24 span<const gidx_t> global_index, const gidx_t global_index_base,
25 span<int> partition, const int partition_base) {
26
27 ATLAS_TRACE("atlas::util::locate_partition");
28 std::size_t size = global_index.size();
29 for (std::size_t j = 0; j < size; ++j) {
30 auto gidx = global_index[j] - global_index_base;
31 ATLAS_ASSERT(gidx < distribution.size());
32 auto p = distribution[gidx] - distribution_base;
33 partition[j] = p + partition_base;
34 }
35 }
36
37 // Given distribution and global_index, find the corresponding partition for each global index
38 // The global-index is typically 1-based
39 void Locator::locate_partition(
40 span<const int> distribution, const int distribution_base,
41 span<const gidx_t> global_index, const gidx_t global_index_base,
42 span<int> partition, const int partition_base) {
43 ATLAS_TRACE("atlas::util::locate_partition");
44 std::size_t size = global_index.size();
45 for (std::size_t j = 0; j < size; ++j) {
46 gidx_t gidx = global_index[j] - global_index_base;
47 ATLAS_ASSERT(gidx < distribution.size());
48 int p = distribution[gidx] - distribution_base;
49 partition[j] = p + partition_base;
50 }
51 }
52
53 // Given global_index, find the corresponding partition and remote_index
54 // This could be costly as it involves memory and communication
55 void Locator::locate_remote_index(
56 const std::string_view mpi_comm,
57 span<const gidx_t> my_glb_idx, const gidx_t my_glb_idx_base, span<const int> my_ghost,
58 span<const gidx_t> global_index, const gidx_t global_index_base,
59 span<const int> partition, const int partition_base,
60 span<idx_t> remote_index, const idx_t remote_index_base) {
61
62 ATLAS_TRACE("atlas::util::locate_remote_index");
63 auto& comm = atlas::mpi::comm(mpi_comm);
64 int mpi_size = comm.size();
65 std::size_t my_size = my_glb_idx.size();
66 std::size_t size = global_index.size();
67 std::vector<std::vector<gidx_t>> recv_gidx(mpi_size);
68 {
69 std::vector<std::vector<gidx_t>> send_gidx(mpi_size);
70 std::vector<std::size_t> send_counts(mpi_size,0);
71 for (std::size_t j=0; j<size; ++j) {
72 int p = partition[j] - partition_base;
73 ++send_counts[p];
74 }
75 for (int p=0; p<mpi_size; ++p) {
76 send_gidx[p].reserve(send_counts[p]);
77 }
78 for (std::size_t j=0; j<size; ++j) {
79 gidx_t gidx = global_index[j] - global_index_base;
80 int p = partition[j] - partition_base;
81 send_gidx[p].emplace_back(gidx);
82 }
83 comm.allToAll(send_gidx, recv_gidx);
84 }
85
86 std::vector<std::vector<idx_t>> recv_ridx(mpi_size);
87 {
88 std::vector<std::vector<idx_t>> send_ridx(mpi_size);
89
90 std::unordered_map<gidx_t, idx_t> gidx_to_ridx;
91 for (idx_t i=0; i<my_size; ++i) {
92 if (not my_ghost[i]) {
93 gidx_to_ridx[my_glb_idx[i] - my_glb_idx_base] = i;
94 }
95 }
96 for( int p=0; p < recv_gidx.size(); ++p ) {
97 for( auto gidx : recv_gidx[p] ) {
98 auto it = gidx_to_ridx.find(gidx);
99 ATLAS_ASSERT( it != gidx_to_ridx.end() );
100 send_ridx[p].emplace_back(it->second);
101 }
102 }
103 comm.allToAll(send_ridx, recv_ridx);
104 }
105 std::vector<std::size_t> c(mpi_size,0);
106 for( idx_t j=0; j<size; ++j) {
107 int p = partition[j] - partition_base;
108 remote_index[j] = recv_ridx[p][c[p]++] + remote_index_base;
109 }
110 }
111
112
113 // Given global_index, find the corresponding partition and remote_index
114 // This could be costly as it involves memory and communication
115 // local information: my_size, my_glb_idx, my_ghost
116 // global information: distribution
117 // requested indices to locate: size, global_index
118 // output of locate: partition, remote_index, remote_index_base
119 void Locator::locate( const std::string_view mpi_comm,
120 span<const gidx_t> my_glb_idx, const gidx_t my_glb_idx_base, span<const int> my_ghost,
121 fspan<const int> distribution, const int distribution_base,
122 span<const gidx_t> global_index, const gidx_t global_index_base,
123 span<int> partition, const int partition_base,
124 span<idx_t> remote_index, const idx_t remote_index_base) {
125
126 ATLAS_TRACE("atlas::util::locate");
127
128 locate_partition(distribution, distribution_base, global_index, global_index_base,
129 partition, partition_base);
130
131 locate_remote_index(mpi_comm, my_glb_idx, my_glb_idx_base, my_ghost,
132 global_index, global_index_base,
133 partition, partition_base,
134 remote_index, remote_index_base);
135 }
136
137 void Locator::locate( const std::string_view mpi_comm,
138 span<const gidx_t> my_glb_idx, const gidx_t my_glb_idx_base, span<const int> my_ghost,
139 span<const int> distribution, const int distribution_base,
140 span<const gidx_t> global_index, const gidx_t global_index_base,
141 span<int> partition, const int partition_base,
142 span<idx_t> remote_index, const idx_t remote_index_base) {
143
144 ATLAS_TRACE("atlas::util::locate");
145
146 locate_partition(distribution, distribution_base,
147 global_index, global_index_base,
148 partition, partition_base);
149
150 locate_remote_index(mpi_comm, my_glb_idx, my_glb_idx_base, my_ghost,
151 global_index, global_index_base,
152 partition, partition_base,
153 remote_index, remote_index_base);
154 }
155
156 } // atlas::parallel