| 1 |
/* |
| 2 |
* (C) Copyright 2013 ECMWF. |
| 3 |
* |
| 4 |
* This software is licensed under the terms of the Apache Licence Version 2.0 |
| 5 |
* which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. |
| 6 |
* In applying this licence, ECMWF does not waive the privileges and immunities |
| 7 |
* granted to it by virtue of its status as an intergovernmental organisation |
| 8 |
* nor does it submit to any jurisdiction. |
| 9 |
*/ |
| 10 |
|
| 11 |
#include <cmath> |
| 12 |
#include <iostream> |
| 13 |
|
| 14 |
#include "atlas/array.h" |
| 15 |
#include "atlas/array/ArrayView.h" |
| 16 |
#include "atlas/array/IndexView.h" |
| 17 |
#include "atlas/mesh/Mesh.h" |
| 18 |
#include "atlas/mesh/Nodes.h" |
| 19 |
#include "atlas/mesh/actions/BuildPeriodicBoundaries.h" |
| 20 |
#include "atlas/parallel/mpi/Statistics.h" |
| 21 |
#include "atlas/parallel/mpi/mpi.h" |
| 22 |
#include "atlas/runtime/Exception.h" |
| 23 |
#include "atlas/util/CoordinateEnums.h" |
| 24 |
#include "atlas/util/LonLatMicroDeg.h" |
| 25 |
#include "atlas/util/PeriodicTransform.h" |
| 26 |
|
| 27 |
using Topology = atlas::mesh::Nodes::Topology; |
| 28 |
using atlas::util::LonLatMicroDeg; |
| 29 |
using atlas::util::PeriodicTransform; |
| 30 |
|
| 31 |
namespace atlas { |
| 32 |
namespace mesh { |
| 33 |
namespace actions { |
| 34 |
|
| 35 |
using uid_t = gidx_t; |
| 36 |
|
| 37 |
void build_periodic_boundaries(Mesh& mesh) { |
| 38 |
ATLAS_TRACE(); |
| 39 |
bool periodic = false; |
| 40 |
mesh.metadata().get("periodic", periodic); |
| 41 |
if (!periodic) { |
| 42 |
mpi::Scope mpi_scope(mesh.mpi_comm()); |
| 43 |
|
| 44 |
auto mpi_size = mpi::size(); |
| 45 |
auto mypart = mpi::rank(); |
| 46 |
|
| 47 |
mesh::Nodes& nodes = mesh.nodes(); |
| 48 |
|
| 49 |
auto flags = array::make_view<int, 1>(nodes.flags()); |
| 50 |
auto ridx = array::make_indexview<idx_t, 1>(nodes.remote_index()); |
| 51 |
auto part = array::make_view<int, 1>(nodes.partition()); |
| 52 |
auto ghost = array::make_view<int, 1>(nodes.ghost()); |
| 53 |
|
| 54 |
int nb_nodes = nodes.size(); |
| 55 |
|
| 56 |
auto xy = array::make_view<double, 2>(nodes.xy()); |
| 57 |
|
| 58 |
// Identify my master and slave nodes on own partition |
| 59 |
// master nodes are at x=0, slave nodes are at x=2pi |
| 60 |
std::map<uid_t, int> master_lookup; |
| 61 |
std::map<uid_t, int> slave_lookup; |
| 62 |
std::vector<int> master_nodes; |
| 63 |
master_nodes.reserve(3 * nb_nodes); |
| 64 |
std::vector<int> slave_nodes; |
| 65 |
slave_nodes.reserve(3 * nb_nodes); |
| 66 |
|
| 67 |
for (idx_t jnode = 0; jnode < nodes.size(); ++jnode) { |
| 68 |
if (Topology::check_all(flags(jnode), Topology::BC | Topology::WEST)) { |
| 69 |
Topology::set(flags(jnode), Topology::PERIODIC); |
| 70 |
if (part(jnode) == mypart) { |
| 71 |
LonLatMicroDeg ll(xy(jnode, XX), xy(jnode, YY)); |
| 72 |
master_lookup[ll.unique()] = jnode; |
| 73 |
master_nodes.push_back(ll.lon()); |
| 74 |
master_nodes.push_back(ll.lat()); |
| 75 |
master_nodes.push_back(jnode); |
| 76 |
} |
| 77 |
} |
| 78 |
else if (Topology::check(flags(jnode), Topology::BC | Topology::EAST)) { |
| 79 |
Topology::set(flags(jnode), Topology::PERIODIC); |
| 80 |
Topology::set(flags(jnode), Topology::GHOST); |
| 81 |
ghost(jnode) = 1; |
| 82 |
LonLatMicroDeg ll(xy(jnode, XX), xy(jnode, YY)); |
| 83 |
slave_lookup[ll.unique()] = jnode; |
| 84 |
slave_nodes.push_back(ll.lon()); |
| 85 |
slave_nodes.push_back(ll.lat()); |
| 86 |
slave_nodes.push_back(jnode); |
| 87 |
ridx(jnode) = -1; |
| 88 |
} |
| 89 |
} |
| 90 |
|
| 91 |
std::vector<std::vector<int>> found_master(mpi_size); |
| 92 |
std::vector<std::vector<int>> send_slave_idx(mpi_size); |
| 93 |
|
| 94 |
// Find masters on other tasks to send to me |
| 95 |
{ |
| 96 |
int sendcnt = slave_nodes.size(); |
| 97 |
std::vector<int> recvcounts(mpi_size); |
| 98 |
|
| 99 |
ATLAS_TRACE_MPI(ALLGATHER) { mpi::comm().allGather(sendcnt, recvcounts.begin(), recvcounts.end()); } |
| 100 |
|
| 101 |
std::vector<int> recvdispls(mpi_size); |
| 102 |
recvdispls[0] = 0; |
| 103 |
int recvcnt = recvcounts[0]; |
| 104 |
for (idx_t jproc = 1; jproc < mpi_size; ++jproc) { |
| 105 |
recvdispls[jproc] = recvdispls[jproc - 1] + recvcounts[jproc - 1]; |
| 106 |
recvcnt += recvcounts[jproc]; |
| 107 |
} |
| 108 |
std::vector<int> recvbuf(recvcnt); |
| 109 |
|
| 110 |
ATLAS_TRACE_MPI(ALLGATHER) { |
| 111 |
mpi::comm().allGatherv(slave_nodes.begin(), slave_nodes.end(), recvbuf.begin(), recvcounts.data(), |
| 112 |
recvdispls.data()); |
| 113 |
} |
| 114 |
|
| 115 |
PeriodicTransform transform; |
| 116 |
for (idx_t jproc = 0; jproc < mpi_size; ++jproc) { |
| 117 |
found_master.reserve(master_nodes.size()); |
| 118 |
send_slave_idx.reserve(master_nodes.size()); |
| 119 |
array::LocalView<int, 2> recv_slave(recvbuf.data() + recvdispls[jproc], |
| 120 |
array::make_shape(recvcounts[jproc] / 3, 3)); |
| 121 |
for (idx_t jnode = 0; jnode < recv_slave.shape(0); ++jnode) { |
| 122 |
LonLatMicroDeg slave(recv_slave(jnode, LON), recv_slave(jnode, LAT)); |
| 123 |
transform(slave, -1); |
| 124 |
uid_t slave_uid = slave.unique(); |
| 125 |
if (master_lookup.count(slave_uid)) { |
| 126 |
int master_idx = master_lookup[slave_uid]; |
| 127 |
int slave_idx = recv_slave(jnode, 2); |
| 128 |
found_master[jproc].push_back(master_idx); |
| 129 |
send_slave_idx[jproc].push_back(slave_idx); |
| 130 |
} |
| 131 |
} |
| 132 |
} |
| 133 |
} |
| 134 |
|
| 135 |
// Fill in data to communicate |
| 136 |
std::vector<std::vector<int>> recv_slave_idx(mpi_size); |
| 137 |
std::vector<std::vector<int>> send_master_part(mpi_size); |
| 138 |
std::vector<std::vector<int>> recv_master_part(mpi_size); |
| 139 |
std::vector<std::vector<int>> send_master_ridx(mpi_size); |
| 140 |
std::vector<std::vector<int>> recv_master_ridx(mpi_size); |
| 141 |
|
| 142 |
// std::vector< std::vector<int> > send_slave_part( mpi_size ); |
| 143 |
// std::vector< std::vector<int> > recv_slave_part( mpi_size ); |
| 144 |
// std::vector< std::vector<int> > send_slave_ridx( mpi_size ); |
| 145 |
// std::vector< std::vector<int> > recv_slave_ridx( mpi_size ); |
| 146 |
|
| 147 |
{ |
| 148 |
for (idx_t jproc = 0; jproc < mpi_size; ++jproc) { |
| 149 |
idx_t nb_found_master = static_cast<idx_t>(found_master[jproc].size()); |
| 150 |
send_master_part[jproc].resize(nb_found_master); |
| 151 |
send_master_ridx[jproc].resize(nb_found_master); |
| 152 |
for (idx_t jnode = 0; jnode < nb_found_master; ++jnode) { |
| 153 |
int loc_idx = found_master[jproc][jnode]; |
| 154 |
send_master_part[jproc][jnode] = part(loc_idx); |
| 155 |
send_master_ridx[jproc][jnode] = loc_idx; |
| 156 |
} |
| 157 |
|
| 158 |
// int nb_found_slaves = found_slave[jproc].size(); |
| 159 |
// send_slave_glb_idx[jproc].resize(nb_found_slaves); |
| 160 |
// send_slave_part [jproc].resize(nb_found_slaves); |
| 161 |
// send_slave_ridx [jproc].resize(nb_found_slaves); |
| 162 |
// for( int jnode=0; jnode<nb_found_slaves; ++jnode ) |
| 163 |
// { |
| 164 |
// int loc_idx = found_slave[jproc][jnode]; |
| 165 |
// send_slave_glb_idx[jproc][jnode] = glb_idx( loc_idx ); |
| 166 |
// send_slave_part [jproc][jnode] = part ( loc_idx ); |
| 167 |
// send_slave_ridx [jproc][jnode] = loc_idx; |
| 168 |
// } |
| 169 |
} |
| 170 |
} |
| 171 |
|
| 172 |
// Communicate |
| 173 |
ATLAS_TRACE_MPI(ALLTOALL) { |
| 174 |
mpi::comm().allToAll(send_slave_idx, recv_slave_idx); |
| 175 |
mpi::comm().allToAll(send_master_part, recv_master_part); |
| 176 |
mpi::comm().allToAll(send_master_ridx, recv_master_ridx); |
| 177 |
// mpi::comm().allToAll( send_slave_part, recv_slave_part ); |
| 178 |
// mpi::comm().allToAll( send_slave_loc, recv_slave_ridx ); |
| 179 |
} |
| 180 |
|
| 181 |
// Fill in periodic |
| 182 |
// unused // int nb_recv_master = 0; |
| 183 |
for (idx_t jproc = 0; jproc < mpi_size; ++jproc) { |
| 184 |
idx_t nb_recv = static_cast<idx_t>(recv_slave_idx[jproc].size()); |
| 185 |
for (idx_t jnode = 0; jnode < nb_recv; ++jnode) { |
| 186 |
idx_t slave_idx = recv_slave_idx[jproc][jnode]; |
| 187 |
part(slave_idx) = recv_master_part[jproc][jnode]; |
| 188 |
ridx(slave_idx) = recv_master_ridx[jproc][jnode]; |
| 189 |
} |
| 190 |
} |
| 191 |
} |
| 192 |
mesh.metadata().set("periodic", true); |
| 193 |
} |
| 194 |
|
| 195 |
// ------------------------------------------------------------------ |
| 196 |
// C wrapper interfaces to C++ routines |
| 197 |
|
| 198 |
void atlas__build_periodic_boundaries(Mesh::Implementation* mesh) { |
| 199 |
ATLAS_ASSERT(mesh != nullptr, "Cannot access uninitialised atlas_Mesh"); |
| 200 |
Mesh m(mesh); |
| 201 |
build_periodic_boundaries(m); |
| 202 |
} |
| 203 |
// ------------------------------------------------------------------ |
| 204 |
|
| 205 |
} // namespace actions |
| 206 |
} // namespace mesh |
| 207 |
} // namespace atlas |