Debian Codegraph @

src/atlas/mesh/actions/BuildPeriodicBoundaries.cc

raw
1 /*
2 * (C) Copyright 2013 ECMWF.
3 *
4 * This software is licensed under the terms of the Apache Licence Version 2.0
5 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
6 * In applying this licence, ECMWF does not waive the privileges and immunities
7 * granted to it by virtue of its status as an intergovernmental organisation
8 * nor does it submit to any jurisdiction.
9 */
10
11 #include <cmath>
12 #include <iostream>
13
14 #include "atlas/array.h"
15 #include "atlas/array/ArrayView.h"
16 #include "atlas/array/IndexView.h"
17 #include "atlas/mesh/Mesh.h"
18 #include "atlas/mesh/Nodes.h"
19 #include "atlas/mesh/actions/BuildPeriodicBoundaries.h"
20 #include "atlas/parallel/mpi/Statistics.h"
21 #include "atlas/parallel/mpi/mpi.h"
22 #include "atlas/runtime/Exception.h"
23 #include "atlas/util/CoordinateEnums.h"
24 #include "atlas/util/LonLatMicroDeg.h"
25 #include "atlas/util/PeriodicTransform.h"
26
27 using Topology = atlas::mesh::Nodes::Topology;
28 using atlas::util::LonLatMicroDeg;
29 using atlas::util::PeriodicTransform;
30
31 namespace atlas {
32 namespace mesh {
33 namespace actions {
34
35 using uid_t = gidx_t;
36
37 void build_periodic_boundaries(Mesh& mesh) {
38 ATLAS_TRACE();
39 bool periodic = false;
40 mesh.metadata().get("periodic", periodic);
41 if (!periodic) {
42 mpi::Scope mpi_scope(mesh.mpi_comm());
43
44 auto mpi_size = mpi::size();
45 auto mypart = mpi::rank();
46
47 mesh::Nodes& nodes = mesh.nodes();
48
49 auto flags = array::make_view<int, 1>(nodes.flags());
50 auto ridx = array::make_indexview<idx_t, 1>(nodes.remote_index());
51 auto part = array::make_view<int, 1>(nodes.partition());
52 auto ghost = array::make_view<int, 1>(nodes.ghost());
53
54 int nb_nodes = nodes.size();
55
56 auto xy = array::make_view<double, 2>(nodes.xy());
57
58 // Identify my master and slave nodes on own partition
59 // master nodes are at x=0, slave nodes are at x=2pi
60 std::map<uid_t, int> master_lookup;
61 std::map<uid_t, int> slave_lookup;
62 std::vector<int> master_nodes;
63 master_nodes.reserve(3 * nb_nodes);
64 std::vector<int> slave_nodes;
65 slave_nodes.reserve(3 * nb_nodes);
66
67 for (idx_t jnode = 0; jnode < nodes.size(); ++jnode) {
68 if (Topology::check_all(flags(jnode), Topology::BC | Topology::WEST)) {
69 Topology::set(flags(jnode), Topology::PERIODIC);
70 if (part(jnode) == mypart) {
71 LonLatMicroDeg ll(xy(jnode, XX), xy(jnode, YY));
72 master_lookup[ll.unique()] = jnode;
73 master_nodes.push_back(ll.lon());
74 master_nodes.push_back(ll.lat());
75 master_nodes.push_back(jnode);
76 }
77 }
78 else if (Topology::check(flags(jnode), Topology::BC | Topology::EAST)) {
79 Topology::set(flags(jnode), Topology::PERIODIC);
80 Topology::set(flags(jnode), Topology::GHOST);
81 ghost(jnode) = 1;
82 LonLatMicroDeg ll(xy(jnode, XX), xy(jnode, YY));
83 slave_lookup[ll.unique()] = jnode;
84 slave_nodes.push_back(ll.lon());
85 slave_nodes.push_back(ll.lat());
86 slave_nodes.push_back(jnode);
87 ridx(jnode) = -1;
88 }
89 }
90
91 std::vector<std::vector<int>> found_master(mpi_size);
92 std::vector<std::vector<int>> send_slave_idx(mpi_size);
93
94 // Find masters on other tasks to send to me
95 {
96 int sendcnt = slave_nodes.size();
97 std::vector<int> recvcounts(mpi_size);
98
99 ATLAS_TRACE_MPI(ALLGATHER) { mpi::comm().allGather(sendcnt, recvcounts.begin(), recvcounts.end()); }
100
101 std::vector<int> recvdispls(mpi_size);
102 recvdispls[0] = 0;
103 int recvcnt = recvcounts[0];
104 for (idx_t jproc = 1; jproc < mpi_size; ++jproc) {
105 recvdispls[jproc] = recvdispls[jproc - 1] + recvcounts[jproc - 1];
106 recvcnt += recvcounts[jproc];
107 }
108 std::vector<int> recvbuf(recvcnt);
109
110 ATLAS_TRACE_MPI(ALLGATHER) {
111 mpi::comm().allGatherv(slave_nodes.begin(), slave_nodes.end(), recvbuf.begin(), recvcounts.data(),
112 recvdispls.data());
113 }
114
115 PeriodicTransform transform;
116 for (idx_t jproc = 0; jproc < mpi_size; ++jproc) {
117 found_master.reserve(master_nodes.size());
118 send_slave_idx.reserve(master_nodes.size());
119 array::LocalView<int, 2> recv_slave(recvbuf.data() + recvdispls[jproc],
120 array::make_shape(recvcounts[jproc] / 3, 3));
121 for (idx_t jnode = 0; jnode < recv_slave.shape(0); ++jnode) {
122 LonLatMicroDeg slave(recv_slave(jnode, LON), recv_slave(jnode, LAT));
123 transform(slave, -1);
124 uid_t slave_uid = slave.unique();
125 if (master_lookup.count(slave_uid)) {
126 int master_idx = master_lookup[slave_uid];
127 int slave_idx = recv_slave(jnode, 2);
128 found_master[jproc].push_back(master_idx);
129 send_slave_idx[jproc].push_back(slave_idx);
130 }
131 }
132 }
133 }
134
135 // Fill in data to communicate
136 std::vector<std::vector<int>> recv_slave_idx(mpi_size);
137 std::vector<std::vector<int>> send_master_part(mpi_size);
138 std::vector<std::vector<int>> recv_master_part(mpi_size);
139 std::vector<std::vector<int>> send_master_ridx(mpi_size);
140 std::vector<std::vector<int>> recv_master_ridx(mpi_size);
141
142 // std::vector< std::vector<int> > send_slave_part( mpi_size );
143 // std::vector< std::vector<int> > recv_slave_part( mpi_size );
144 // std::vector< std::vector<int> > send_slave_ridx( mpi_size );
145 // std::vector< std::vector<int> > recv_slave_ridx( mpi_size );
146
147 {
148 for (idx_t jproc = 0; jproc < mpi_size; ++jproc) {
149 idx_t nb_found_master = static_cast<idx_t>(found_master[jproc].size());
150 send_master_part[jproc].resize(nb_found_master);
151 send_master_ridx[jproc].resize(nb_found_master);
152 for (idx_t jnode = 0; jnode < nb_found_master; ++jnode) {
153 int loc_idx = found_master[jproc][jnode];
154 send_master_part[jproc][jnode] = part(loc_idx);
155 send_master_ridx[jproc][jnode] = loc_idx;
156 }
157
158 // int nb_found_slaves = found_slave[jproc].size();
159 // send_slave_glb_idx[jproc].resize(nb_found_slaves);
160 // send_slave_part [jproc].resize(nb_found_slaves);
161 // send_slave_ridx [jproc].resize(nb_found_slaves);
162 // for( int jnode=0; jnode<nb_found_slaves; ++jnode )
163 // {
164 // int loc_idx = found_slave[jproc][jnode];
165 // send_slave_glb_idx[jproc][jnode] = glb_idx( loc_idx );
166 // send_slave_part [jproc][jnode] = part ( loc_idx );
167 // send_slave_ridx [jproc][jnode] = loc_idx;
168 // }
169 }
170 }
171
172 // Communicate
173 ATLAS_TRACE_MPI(ALLTOALL) {
174 mpi::comm().allToAll(send_slave_idx, recv_slave_idx);
175 mpi::comm().allToAll(send_master_part, recv_master_part);
176 mpi::comm().allToAll(send_master_ridx, recv_master_ridx);
177 // mpi::comm().allToAll( send_slave_part, recv_slave_part );
178 // mpi::comm().allToAll( send_slave_loc, recv_slave_ridx );
179 }
180
181 // Fill in periodic
182 // unused // int nb_recv_master = 0;
183 for (idx_t jproc = 0; jproc < mpi_size; ++jproc) {
184 idx_t nb_recv = static_cast<idx_t>(recv_slave_idx[jproc].size());
185 for (idx_t jnode = 0; jnode < nb_recv; ++jnode) {
186 idx_t slave_idx = recv_slave_idx[jproc][jnode];
187 part(slave_idx) = recv_master_part[jproc][jnode];
188 ridx(slave_idx) = recv_master_ridx[jproc][jnode];
189 }
190 }
191 }
192 mesh.metadata().set("periodic", true);
193 }
194
195 // ------------------------------------------------------------------
196 // C wrapper interfaces to C++ routines
197
198 void atlas__build_periodic_boundaries(Mesh::Implementation* mesh) {
199 ATLAS_ASSERT(mesh != nullptr, "Cannot access uninitialised atlas_Mesh");
200 Mesh m(mesh);
201 build_periodic_boundaries(m);
202 }
203 // ------------------------------------------------------------------
204
205 } // namespace actions
206 } // namespace mesh
207 } // namespace atlas