| 1 |
/* |
| 2 |
* (C) Copyright 2013 ECMWF. |
| 3 |
* |
| 4 |
* This software is licensed under the terms of the Apache Licence Version 2.0 |
| 5 |
* which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. |
| 6 |
* In applying this licence, ECMWF does not waive the privileges and immunities |
| 7 |
* granted to it by virtue of its status as an intergovernmental organisation |
| 8 |
* nor does it submit to any jurisdiction. |
| 9 |
*/ |
| 10 |
#include "atlas/functionspace/StructuredColumns.h" |
| 11 |
|
| 12 |
#include <functional> |
| 13 |
#include <iomanip> |
| 14 |
#include <new> // for bad_alloc exception |
| 15 |
#include <sstream> |
| 16 |
#include <string> |
| 17 |
#include <unordered_map> |
| 18 |
|
| 19 |
#include "eckit/log/Bytes.h" |
| 20 |
|
| 21 |
#include "atlas/array/MakeView.h" |
| 22 |
#include "atlas/grid/StructuredGrid.h" |
| 23 |
#include "atlas/library/Library.h" |
| 24 |
#include "atlas/mesh/Mesh.h" |
| 25 |
#include "atlas/parallel/GatherScatter.h" |
| 26 |
#include "atlas/parallel/mpi/Statistics.h" |
| 27 |
#include "atlas/parallel/mpi/mpi.h" |
| 28 |
#include "atlas/runtime/Exception.h" |
| 29 |
#include "atlas/runtime/Log.h" |
| 30 |
#include "atlas/runtime/Trace.h" |
| 31 |
#include "atlas/util/vector.h" |
| 32 |
|
| 33 |
namespace atlas { |
| 34 |
namespace functionspace { |
| 35 |
namespace detail { |
| 36 |
|
| 37 |
void StructuredColumns::create_remote_index() const { |
| 38 |
field_remote_index_ = Field("remote_idx", array::make_datatype<idx_t>(), array::make_shape(size_halo_)); |
| 39 |
auto remote_idx = array::make_indexview<idx_t, 1>(field_remote_index_); |
| 40 |
atlas_omp_parallel_for(idx_t n = 0; n < size_owned_; ++n) { remote_idx(n) = n; } |
| 41 |
|
| 42 |
|
| 43 |
ATLAS_TRACE_SCOPE("Parallelisation ...") { |
| 44 |
auto build_partition_graph = [this]() -> std::unique_ptr<Mesh::PartitionGraph> { |
| 45 |
const eckit::mpi::Comm& comm = mpi::comm(mpi_comm()); |
| 46 |
const int mpi_size = int(comm.size()); |
| 47 |
const int mpi_rank = int(comm.rank()); |
| 48 |
|
| 49 |
auto part = array::make_view<int, 1>(this->partition()); |
| 50 |
|
| 51 |
std::vector<int> others_set(mpi_size, 0); |
| 52 |
others_set[mpi_rank] = 1; |
| 53 |
for (idx_t i = size_owned_; i < size_halo_; ++i) { |
| 54 |
others_set[part(i)] = 1; // present |
| 55 |
} |
| 56 |
std::vector<int> others; |
| 57 |
others.reserve(mpi_size); |
| 58 |
for (idx_t p = 0; p < mpi_size; ++p) { |
| 59 |
if (others_set[p]) { |
| 60 |
others.emplace_back(p); |
| 61 |
} |
| 62 |
} |
| 63 |
|
| 64 |
eckit::mpi::Buffer<int> recv_others(mpi_size); |
| 65 |
|
| 66 |
ATLAS_TRACE_MPI(ALLGATHER) { comm.allGatherv(others.begin(), others.end(), recv_others); } |
| 67 |
|
| 68 |
std::vector<idx_t> counts(recv_others.counts.begin(), recv_others.counts.end()); |
| 69 |
std::vector<idx_t> displs(recv_others.displs.begin(), recv_others.displs.end()); |
| 70 |
std::vector<idx_t> values(recv_others.buffer.begin(), recv_others.buffer.end()); |
| 71 |
return std::unique_ptr<Mesh::PartitionGraph>( |
| 72 |
new Mesh::PartitionGraph(values.data(), mpi_size, displs.data(), counts.data())); |
| 73 |
}; |
| 74 |
|
| 75 |
std::unique_ptr<Mesh::PartitionGraph> graph_ptr; |
| 76 |
ATLAS_TRACE_SCOPE("Building partition graph...") { graph_ptr = build_partition_graph(); } |
| 77 |
const Mesh::PartitionGraph& graph = *graph_ptr; |
| 78 |
|
| 79 |
ATLAS_TRACE_SCOPE("Setup remote_index fields...") { |
| 80 |
auto p = array::make_view<int, 1>(partition()); |
| 81 |
auto g = array::make_view<gidx_t, 1>(global_index()); |
| 82 |
|
| 83 |
const eckit::mpi::Comm& comm = mpi::comm(mpi_comm()); |
| 84 |
const int mpi_rank = int(comm.rank()); |
| 85 |
|
| 86 |
auto neighbours = graph.nearestNeighbours(mpi_rank); |
| 87 |
const idx_t nb_neighbours = static_cast<idx_t>(neighbours.size()); |
| 88 |
std::unordered_map<int, idx_t> part_to_neighbour; |
| 89 |
ATLAS_TRACE_SCOPE("part_to_neighbour") { |
| 90 |
part_to_neighbour.reserve(nb_neighbours); |
| 91 |
for (idx_t j = 0; j < nb_neighbours; ++j) { |
| 92 |
part_to_neighbour[neighbours[j]] = j; |
| 93 |
} |
| 94 |
} |
| 95 |
std::vector<idx_t> halo_per_neighbour(neighbours.size(), 0); |
| 96 |
ATLAS_TRACE_SCOPE("set halo_per_neighbour") |
| 97 |
for (idx_t i = size_owned_; i < size_halo_; ++i) { |
| 98 |
halo_per_neighbour[part_to_neighbour[p(i)]]++; |
| 99 |
} |
| 100 |
|
| 101 |
std::vector<std::vector<gidx_t>> g_per_neighbour(neighbours.size()); |
| 102 |
ATLAS_TRACE_SCOPE("assemble g_per_neighbour") { |
| 103 |
for (idx_t j = 0; j < nb_neighbours; ++j) { |
| 104 |
g_per_neighbour[j].reserve(halo_per_neighbour[j]); |
| 105 |
} |
| 106 |
for (idx_t j = size_owned_; j < size_halo_; ++j) { |
| 107 |
g_per_neighbour[part_to_neighbour[p(j)]].emplace_back(g(j)); |
| 108 |
} |
| 109 |
} |
| 110 |
|
| 111 |
std::vector<eckit::mpi::Request> send_requests(neighbours.size()); |
| 112 |
std::vector<eckit::mpi::Request> recv_requests(neighbours.size()); |
| 113 |
|
| 114 |
std::vector<idx_t> recv_size(neighbours.size()); |
| 115 |
std::vector<idx_t> send_size(neighbours.size()); |
| 116 |
|
| 117 |
int tag = 0; |
| 118 |
ATLAS_TRACE_SCOPE("send-receive g_per_neighbour size") { |
| 119 |
for (idx_t j = 0; j < nb_neighbours; ++j) { |
| 120 |
send_size[j] = static_cast<idx_t>(g_per_neighbour[j].size()); |
| 121 |
send_requests[j] = comm.iSend(send_size[j], neighbours[j], tag); |
| 122 |
recv_requests[j] = comm.iReceive(recv_size[j], neighbours[j], tag); |
| 123 |
} |
| 124 |
|
| 125 |
ATLAS_TRACE_MPI(WAIT) { |
| 126 |
for (idx_t j = 0; j < nb_neighbours; ++j) { |
| 127 |
comm.wait(send_requests[j]); |
| 128 |
} |
| 129 |
|
| 130 |
for (idx_t j = 0; j < nb_neighbours; ++j) { |
| 131 |
comm.wait(recv_requests[j]); |
| 132 |
} |
| 133 |
} |
| 134 |
} |
| 135 |
|
| 136 |
std::vector<std::vector<gidx_t>> recv_g_per_neighbour(neighbours.size()); |
| 137 |
ATLAS_TRACE_SCOPE("send-receive g_per_neighbour") |
| 138 |
for (idx_t j = 0; j < nb_neighbours; ++j) { |
| 139 |
recv_g_per_neighbour[j].resize(recv_size[j]); |
| 140 |
|
| 141 |
send_requests[j] = comm.iSend(g_per_neighbour[j].data(), g_per_neighbour[j].size(), neighbours[j], tag); |
| 142 |
recv_requests[j] = |
| 143 |
comm.iReceive(recv_g_per_neighbour[j].data(), recv_g_per_neighbour[j].size(), neighbours[j], tag); |
| 144 |
} |
| 145 |
|
| 146 |
std::vector<std::vector<idx_t>> send_r_per_neighbour(neighbours.size()); |
| 147 |
|
| 148 |
// Assemble "send_r_per_neighbour" |
| 149 |
// Depending if we can afford memory for a globally sized array, we can have a |
| 150 |
// much faster version of g_to_r map using std::vector. |
| 151 |
// TODO: using c++14 we can create a polymorphic lambda to avoid duplicated |
| 152 |
// code in the two branches of following if. |
| 153 |
|
| 154 |
size_t max_glb_idx = grid().size(); |
| 155 |
atlas::vector<idx_t> g_to_r_vector; |
| 156 |
bool use_unordered_map_fallback = false; |
| 157 |
try { |
| 158 |
g_to_r_vector.resize(max_glb_idx + 1); |
| 159 |
} |
| 160 |
catch (std::bad_alloc& e) { |
| 161 |
if (comm.size() > 1) { |
| 162 |
Log::warning() << "Could not allocate " << eckit::Bytes((max_glb_idx + 1) * sizeof(idx_t)) << Here() |
| 163 |
<< "\n" |
| 164 |
<< "Using slower unordered_map fallback to map global to remote indices" |
| 165 |
<< std::endl; |
| 166 |
use_unordered_map_fallback = true; |
| 167 |
} |
| 168 |
else { |
| 169 |
throw_Exception( |
| 170 |
"Could not allocate " + std::string(eckit::Bytes((max_glb_idx + 1) * sizeof(idx_t))), Here()); |
| 171 |
} |
| 172 |
} |
| 173 |
if (not use_unordered_map_fallback) { |
| 174 |
auto& g_to_r = g_to_r_vector; |
| 175 |
ATLAS_TRACE_SCOPE("g_to_r (using vector)") { |
| 176 |
atlas_omp_parallel_for(idx_t j = 0; j < size_owned_; ++j) { |
| 177 |
// parallel omp possible for ``` g_to_r[g(j)] = j ``` as we only loop over size_owned, |
| 178 |
// where global_index is such that race-conditions cannot occur |
| 179 |
#if ATLAS_ARRAYVIEW_BOUNDS_CHECKING |
| 180 |
if (g(j) >= max_glb_idx + 1) { |
| 181 |
ATLAS_DEBUG_VAR(g(j)); |
| 182 |
throw_OutOfRange("g_to_r", g(j), max_glb_idx + 1, Here()); |
| 183 |
} |
| 184 |
#endif |
| 185 |
g_to_r[g(j)] = j; |
| 186 |
} |
| 187 |
} |
| 188 |
for (idx_t j = 0; j < nb_neighbours; ++j) { |
| 189 |
send_r_per_neighbour[j].reserve(recv_size[j]); |
| 190 |
|
| 191 |
comm.wait(recv_requests[j]); // wait for recv_g_per_neighbour[j] |
| 192 |
for (gidx_t gidx : recv_g_per_neighbour[j]) { |
| 193 |
send_r_per_neighbour[j].emplace_back(g_to_r[gidx]); |
| 194 |
} |
| 195 |
} |
| 196 |
} |
| 197 |
else { |
| 198 |
std::unordered_map<gidx_t, idx_t> g_to_r; |
| 199 |
g_to_r.reserve(size_owned_); |
| 200 |
|
| 201 |
ATLAS_TRACE_SCOPE("g_to_r (using unordered set)") { |
| 202 |
for (idx_t j = 0; j < size_owned_; ++j) { |
| 203 |
g_to_r[g(j)] = j; |
| 204 |
} |
| 205 |
} |
| 206 |
ATLAS_TRACE_SCOPE("assemble r_per_neighbour") |
| 207 |
for (idx_t j = 0; j < nb_neighbours; ++j) { |
| 208 |
send_r_per_neighbour[j].reserve(recv_size[j]); |
| 209 |
|
| 210 |
comm.wait(recv_requests[j]); // wait for recv_g_per_neighbour[j] |
| 211 |
for (gidx_t gidx : recv_g_per_neighbour[j]) { |
| 212 |
send_r_per_neighbour[j].emplace_back(g_to_r[gidx]); |
| 213 |
} |
| 214 |
} |
| 215 |
} |
| 216 |
|
| 217 |
std::vector<std::vector<idx_t>> r_per_neighbour(neighbours.size()); |
| 218 |
|
| 219 |
ATLAS_TRACE_SCOPE("send-receive r_per_neighbour") { |
| 220 |
for (idx_t j = 0; j < nb_neighbours; ++j) { |
| 221 |
r_per_neighbour[j].resize(halo_per_neighbour[j]); |
| 222 |
} |
| 223 |
|
| 224 |
ATLAS_TRACE_MPI(ALLTOALL) { |
| 225 |
for (idx_t j = 0; j < nb_neighbours; ++j) { |
| 226 |
comm.wait(send_requests[j]); |
| 227 |
send_requests[j] = comm.iSend(send_r_per_neighbour[j].data(), send_r_per_neighbour[j].size(), |
| 228 |
neighbours[j], tag); |
| 229 |
recv_requests[j] = |
| 230 |
comm.iReceive(r_per_neighbour[j].data(), r_per_neighbour[j].size(), neighbours[j], tag); |
| 231 |
} |
| 232 |
} |
| 233 |
ATLAS_TRACE_MPI(WAIT) { |
| 234 |
for (idx_t j = 0; j < nb_neighbours; ++j) { |
| 235 |
comm.wait(recv_requests[j]); |
| 236 |
} |
| 237 |
} |
| 238 |
} |
| 239 |
|
| 240 |
|
| 241 |
std::vector<idx_t> counters(neighbours.size(), 0); |
| 242 |
ATLAS_TRACE_SCOPE("set remote_idx") |
| 243 |
for (idx_t j = size_owned_; j < size_halo_; ++j) { |
| 244 |
idx_t neighbour = part_to_neighbour[p(j)]; |
| 245 |
remote_idx(j) = r_per_neighbour[neighbour][counters[neighbour]++]; |
| 246 |
} |
| 247 |
|
| 248 |
ATLAS_TRACE_MPI(WAIT) { |
| 249 |
for (idx_t j = 0; j < nb_neighbours; ++j) { |
| 250 |
comm.wait(send_requests[j]); |
| 251 |
} |
| 252 |
} |
| 253 |
} |
| 254 |
} |
| 255 |
} |
| 256 |
|
| 257 |
} // namespace detail |
| 258 |
|
| 259 |
// ---------------------------------------------------------------------------- |
| 260 |
|
| 261 |
} // namespace functionspace |
| 262 |
} // namespace atlas |