Debian Codegraph @

src/atlas/mesh/actions/BuildParallelFields.cc

raw
1 /*
2 * (C) Copyright 2013 ECMWF.
3 *
4 * This software is licensed under the terms of the Apache Licence Version 2.0
5 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
6 * In applying this licence, ECMWF does not waive the privileges and immunities
7 * granted to it by virtue of its status as an intergovernmental organisation
8 * nor does it submit to any jurisdiction.
9 */
10
11
12 #include <cmath>
13 #include <iomanip>
14 #include <iostream>
15 #include <stdexcept>
16
17 #include "atlas/array.h"
18 #include "atlas/array/ArrayView.h"
19 #include "atlas/array/IndexView.h"
20 #include "atlas/field/Field.h"
21 #include "atlas/mesh/HybridElements.h"
22 #include "atlas/mesh/Mesh.h"
23 #include "atlas/mesh/Nodes.h"
24 #include "atlas/mesh/actions/BuildParallelFields.h"
25 #include "atlas/parallel/GatherScatter.h"
26 #include "atlas/parallel/mpi/Buffer.h"
27 #include "atlas/parallel/mpi/mpi.h"
28 #include "atlas/runtime/Exception.h"
29 #include "atlas/runtime/Log.h"
30 #include "atlas/runtime/Trace.h"
31 #include "atlas/util/CoordinateEnums.h"
32 #include "atlas/util/PeriodicTransform.h"
33 #include "atlas/util/Unique.h"
34
35 #define EDGE(jedge) \
36 "Edge(" << node_gidx(edge_nodes(jedge, 0)) << "[p" << node_part(edge_nodes(jedge, 0)) << "] " \
37 << node_gidx(edge_nodes(jedge, 1)) << "[p" << node_part(edge_nodes(jedge, 1)) << "])"
38
39 //#define DEBUGGING_PARFIELDS
40 #ifdef DEBUGGING_PARFIELDS
41 #define own1 2419089
42 #define own2 2423185
43 #define OWNED_EDGE(jedge) \
44 ((node_gidx(edge_nodes(jedge, 0)) == own1 && node_gidx(edge_nodes(jedge, 1)) == own2) || \
45 (node_gidx(edge_nodes(jedge, 0)) == own2 && node_gidx(edge_nodes(jedge, 1)) == own1))
46 #define per1 -1
47 #define per2 -1
48 #define PERIODIC_EDGE(jedge) \
49 ((node_gidx(edge_nodes(jedge, 0)) == per1 && node_gidx(edge_nodes(jedge, 1)) == per2) || \
50 (node_gidx(edge_nodes(jedge, 0)) == per2 && node_gidx(edge_nodes(jedge, 1)) == per1))
51 #define find1 39 // 1697
52 #define find2 41 // 1698
53 #define FIND_EDGE(jedge) \
54 ((node_gidx(edge_nodes(jedge, 0)) == find1 && node_gidx(edge_nodes(jedge, 1)) == find2) || \
55 (node_gidx(edge_nodes(jedge, 0)) == find2 && node_gidx(edge_nodes(jedge, 1)) == find1))
56 #define find_gidx 689849552510167040
57 #define FIND_GIDX(UID) ((UID) == find_gidx)
58 #endif
59
60 using Topology = atlas::mesh::Nodes::Topology;
61 using atlas::util::PeriodicTransform;
62 using atlas::util::UniqueLonLat;
63
64 namespace atlas {
65 namespace mesh {
66 namespace actions {
67
68 Field& build_nodes_partition(mesh::Nodes& nodes);
69 Field& build_nodes_remote_idx(mesh::Nodes& nodes);
70 Field& build_nodes_global_idx(mesh::Nodes& nodes);
71 Field& build_edges_partition(Mesh& mesh);
72 Field& build_edges_remote_idx(Mesh& mesh);
73 Field& build_edges_global_idx(Mesh& mesh);
74
75 //----------------------------------------------------------------------------------------------------------------------
76
77 using uid_t = gidx_t;
78
79 namespace {
80
81 struct Node {
82 Node(gidx_t gid, idx_t idx) {
83 g = gid;
84 i = idx;
85 }
86 gidx_t g;
87 idx_t i;
88 bool operator<(const Node& other) const { return (g < other.g); }
89 };
90
91 } // namespace
92
93 //----------------------------------------------------------------------------------------------------------------------
94
95 void build_parallel_fields(Mesh& mesh) {
96 ATLAS_TRACE();
97 build_nodes_parallel_fields(mesh);
98 }
99
100 //----------------------------------------------------------------------------------------------------------------------
101
102 void build_nodes_parallel_fields(Mesh& mesh) {
103 mpi::Scope mpi_scope(mesh.mpi_comm());
104 build_nodes_parallel_fields(mesh.nodes());
105 }
106
107 void build_nodes_parallel_fields(mesh::Nodes& nodes) {
108 ATLAS_TRACE();
109 bool parallel = false;
110 nodes.metadata().get("parallel", parallel);
111 if (!parallel) {
112 build_nodes_partition(nodes);
113 build_nodes_remote_idx(nodes);
114 build_nodes_global_idx(nodes);
115 }
116 nodes.metadata().set("parallel", true);
117 }
118
119 //----------------------------------------------------------------------------------------------------------------------
120
121 void build_edges_parallel_fields(Mesh& mesh) {
122 ATLAS_TRACE();
123 mpi::Scope mpi_scope(mesh.mpi_comm());
124 build_edges_partition(mesh);
125 build_edges_remote_idx(mesh);
126 /*
127 * We turn following off. It is expensive and we don't really care about a nice
128 * contiguous
129 * ordering.
130 */
131 // build_edges_global_idx( mesh );
132 }
133
134 //----------------------------------------------------------------------------------------------------------------------
135
136 Field& build_nodes_global_idx(mesh::Nodes& nodes) {
137 ATLAS_TRACE();
138
139 array::ArrayView<gidx_t, 1> glb_idx = array::make_view<gidx_t, 1>(nodes.global_index());
140
141 UniqueLonLat compute_uid(nodes);
142
143 for (idx_t jnode = 0; jnode < glb_idx.shape(0); ++jnode) {
144 if (glb_idx(jnode) <= 0) {
145 glb_idx(jnode) = compute_uid(jnode);
146 }
147 }
148 return nodes.global_index();
149 }
150
151 void renumber_nodes_glb_idx(Mesh& mesh) {
152 mpi::Scope mpi_scope(mesh.mpi_comm());
153 renumber_nodes_glb_idx(mesh.nodes());
154 }
155
156 void renumber_nodes_glb_idx(mesh::Nodes& nodes) {
157 bool human_readable(false);
158 nodes.global_index().metadata().get("human_readable", human_readable);
159 if (human_readable) {
160 /* nothing to be done */
161 return;
162 }
163
164 ATLAS_TRACE();
165
166 // TODO: ATLAS-14: fix renumbering of EAST periodic boundary points
167 // --> Those specific periodic points at the EAST boundary are not checked for
168 // uid,
169 // and could receive different gidx for different tasks
170
171 UniqueLonLat compute_uid(nodes);
172
173 // unused // int mypart = mpi::rank();
174 int nparts = mpi::size();
175 idx_t root = 0;
176
177 array::ArrayView<gidx_t, 1> glb_idx = array::make_view<gidx_t, 1>(nodes.global_index());
178
179 /*
180 * Sorting following gidx will define global order of
181 * gathered fields. Special care needs to be taken for
182 * pole edges, as their centroid might coincide with
183 * other edges
184 */
185 int nb_nodes = glb_idx.shape(0);
186 for (int jnode = 0; jnode < nb_nodes; ++jnode) {
187 if (glb_idx(jnode) <= 0) {
188 glb_idx(jnode) = compute_uid(jnode);
189 }
190 }
191
192 // 1) Gather all global indices, together with location
193 array::ArrayT<uid_t> loc_id_arr(nb_nodes);
194 array::ArrayView<uid_t, 1> loc_id = array::make_view<uid_t, 1>(loc_id_arr);
195
196 for (int jnode = 0; jnode < nb_nodes; ++jnode) {
197 loc_id(jnode) = glb_idx(jnode);
198 }
199
200 std::vector<int> recvcounts(mpi::size());
201 std::vector<int> recvdispls(mpi::size());
202
203 ATLAS_TRACE_MPI(GATHER) { mpi::comm().gather(nb_nodes, recvcounts, root); }
204
205 recvdispls[0] = 0;
206 for (int jpart = 1; jpart < nparts; ++jpart) { // start at 1
207 recvdispls[jpart] = recvcounts[jpart - 1] + recvdispls[jpart - 1];
208 }
209 int glb_nb_nodes = std::accumulate(recvcounts.begin(), recvcounts.end(), 0);
210
211 array::ArrayT<uid_t> glb_id_arr(glb_nb_nodes);
212 array::ArrayView<uid_t, 1> glb_id = array::make_view<uid_t, 1>(glb_id_arr);
213
214 ATLAS_TRACE_MPI(GATHER) {
215 mpi::comm().gatherv(loc_id.data(), loc_id.size(), glb_id.data(), recvcounts.data(), recvdispls.data(), root);
216 }
217
218 // 2) Sort all global indices, and renumber from 1 to glb_nb_edges
219 std::vector<Node> node_sort;
220 node_sort.reserve(glb_nb_nodes);
221 ATLAS_TRACE_SCOPE("sort global indices") {
222 for (idx_t jnode = 0; jnode < glb_id.shape(0); ++jnode) {
223 node_sort.emplace_back(glb_id(jnode), jnode);
224 }
225 std::sort(node_sort.begin(), node_sort.end());
226 }
227
228 // Assume edge gid start
229 uid_t gid = 0;
230 const idx_t nb_sorted_nodes = static_cast<idx_t>(node_sort.size());
231 for (idx_t jnode = 0; jnode < nb_sorted_nodes; ++jnode) {
232 if (jnode == 0) {
233 ++gid;
234 }
235 else if (node_sort[jnode].g != node_sort[jnode - 1].g) {
236 ++gid;
237 }
238 idx_t inode = node_sort[jnode].i;
239 glb_id(inode) = gid;
240 }
241
242 // 3) Scatter renumbered back
243 ATLAS_TRACE_MPI(SCATTER) {
244 mpi::comm().scatterv(glb_id.data(), recvcounts.data(), recvdispls.data(), loc_id.data(), loc_id.size(), root);
245 }
246
247 for (int jnode = 0; jnode < nb_nodes; ++jnode) {
248 glb_idx(jnode) = loc_id(jnode);
249 }
250 nodes.global_index().metadata().set("human_readable", true);
251 }
252
253 //----------------------------------------------------------------------------------------------------------------------
254
255 Field& build_nodes_remote_idx(mesh::Nodes& nodes) {
256 ATLAS_TRACE();
257 idx_t mypart = static_cast<idx_t>(mpi::rank());
258 idx_t nparts = static_cast<idx_t>(mpi::size());
259
260
261 std::vector<idx_t> proc(nparts);
262 for (idx_t jpart = 0; jpart < nparts; ++jpart) {
263 proc[jpart] = jpart;
264 }
265
266 auto ridx = array::make_indexview<idx_t, 1>(nodes.remote_index());
267 const auto part = array::make_view<int, 1>(nodes.partition());
268 const auto flags = array::make_view<int, 1>(nodes.flags());
269
270 const PeriodicTransform transform_periodic_east(-360.);
271 const PeriodicTransform transform_periodic_west(+360.);
272 const UniqueLonLat compute_uid_lonlat(nodes);
273
274 auto compute_uid = [&](idx_t jnode) {
275 constexpr int PERIODIC = util::Topology::PERIODIC;
276 constexpr int EAST = util::Topology::EAST;
277 constexpr int WEST = util::Topology::WEST;
278 const auto flags_view = util::Bitflags::view(flags(jnode));
279 if (flags_view.check(PERIODIC | EAST)) {
280 return compute_uid_lonlat(jnode, transform_periodic_east);
281 }
282 if (flags_view.check(PERIODIC | WEST)) {
283 return compute_uid_lonlat(jnode, transform_periodic_west);
284 }
285 return compute_uid_lonlat(jnode);
286 };
287
288 idx_t nb_nodes = nodes.size();
289
290 constexpr idx_t varsize = 2;
291
292 std::vector<std::vector<uid_t>> send_needed(mpi::size());
293 std::vector<std::vector<uid_t>> recv_needed(mpi::size());
294 std::map<uid_t, int> lookup;
295 for (idx_t jnode = 0; jnode < nb_nodes; ++jnode) {
296 uid_t uid = compute_uid(jnode);
297
298 if (idx_t(part(jnode)) == mypart) {
299 lookup[uid] = jnode;
300 ridx(jnode) = jnode;
301 }
302 else {
303 ATLAS_ASSERT(jnode < part.shape(0));
304 if (part(jnode) >= static_cast<int>(proc.size())) {
305 std::stringstream msg;
306 msg << "Assertion [part(" << jnode << ") < proc.size()] failed\n"
307 << "part(" << jnode << ") = " << part(jnode) << "\n"
308 << "proc.size() = " << proc.size();
309 throw_AssertionFailed(msg.str(), Here());
310 }
311 ATLAS_ASSERT(part(jnode) < (idx_t)proc.size());
312 ATLAS_ASSERT((size_t)proc[part(jnode)] < send_needed.size());
313 send_needed[proc[part(jnode)]].push_back(uid);
314 send_needed[proc[part(jnode)]].push_back(jnode);
315 }
316 }
317
318 ATLAS_TRACE_MPI(ALLTOALL) { mpi::comm().allToAll(send_needed, recv_needed); }
319
320 std::vector<std::vector<int>> send_found(mpi::size());
321 std::vector<std::vector<int>> recv_found(mpi::size());
322
323 for (idx_t jpart = 0; jpart < nparts; ++jpart) {
324 const std::vector<uid_t>& recv_node = recv_needed[proc[jpart]];
325 const idx_t nb_recv_nodes = idx_t(recv_node.size()) / varsize;
326 for (idx_t jnode = 0; jnode < nb_recv_nodes; ++jnode) {
327 uid_t uid = recv_node[jnode * varsize + 0];
328 int inode = recv_node[jnode * varsize + 1];
329 send_found[proc[jpart]].push_back(inode);
330 send_found[proc[jpart]].push_back(lookup.count(uid) ? lookup[uid] : -1);
331 }
332 }
333
334 ATLAS_TRACE_MPI(ALLTOALL) { mpi::comm().allToAll(send_found, recv_found); }
335
336 std::stringstream errstream;
337 size_t failed{0};
338 const auto gidx = array::make_view<gidx_t, 1>(nodes.global_index());
339 for (idx_t jpart = 0; jpart < nparts; ++jpart) {
340 const std::vector<int>& recv_node = recv_found[proc[jpart]];
341 const idx_t nb_recv_nodes = recv_node.size() / 2;
342 // array::ArrayView<int,2> recv_node( recv_found[ proc[jpart] ].data(),
343 // array::make_shape(recv_found[ proc[jpart] ].size()/2,2) );
344 for (idx_t jnode = 0; jnode < nb_recv_nodes; ++jnode) {
345 idx_t inode = recv_node[jnode * 2 + 0];
346 idx_t ridx_inode = recv_node[jnode * 2 + 1];
347 if (ridx_inode >= 0) {
348 ridx(recv_node[jnode * 2 + 0]) = ridx_inode;
349 }
350 else {
351 ++failed;
352 errstream << "\n[" << mpi::rank() << "] "
353 << "Node with global index " << gidx(inode) << " not found on part [" << part(inode) << "]";
354 }
355 }
356 }
357
358 if (failed) {
359 throw_AssertionFailed(errstream.str(), Here());
360 }
361
362 return nodes.remote_index();
363 }
364
365 //----------------------------------------------------------------------------------------------------------------------
366
367 Field& build_nodes_partition(mesh::Nodes& nodes) {
368 ATLAS_TRACE();
369 return nodes.partition();
370 }
371
372 //----------------------------------------------------------------------------------------------------------------------
373
374 Field& build_edges_partition(Mesh& mesh) {
375 ATLAS_TRACE();
376
377 const mesh::Nodes& nodes = mesh.nodes();
378
379 idx_t mypart = mpi::rank();
380
381 mesh::HybridElements& edges = mesh.edges();
382 auto edge_part = array::make_view<int, 1>(edges.partition());
383 //const auto edge_glb_idx = array::make_view<gidx_t, 1>( edges.global_index() );
384
385 const auto& edge_nodes = edges.node_connectivity();
386 const auto& edge_to_elem = edges.cell_connectivity();
387
388 const auto node_part = array::make_view<int, 1>(nodes.partition());
389 const auto xy = array::make_view<double, 2>(nodes.xy());
390 const auto flags = array::make_view<int, 1>(nodes.flags());
391 const auto node_gidx = array::make_view<gidx_t, 1>(nodes.global_index());
392
393 const auto elem_gidx = array::make_view<gidx_t, 1>(mesh.cells().global_index());
394 const auto elem_part = array::make_view<int, 1>(mesh.cells().partition());
395 //const auto elem_halo = array::make_view<int, 1>( mesh.cells().halo() );
396
397 auto check_flags = [&](idx_t jedge, int flag) {
398 idx_t ip1 = edge_nodes(jedge, 0);
399 idx_t ip2 = edge_nodes(jedge, 1);
400 return Topology::check(flags(ip1), flag) && Topology::check(flags(ip2), flag);
401 };
402 auto domain_bdry = [&](idx_t jedge) {
403 if (check_flags(jedge, Topology::BC | Topology::NORTH)) {
404 return true;
405 }
406 if (check_flags(jedge, Topology::BC | Topology::SOUTH)) {
407 return true;
408 }
409 if (check_flags(jedge, Topology::BC | Topology::WEST)) {
410 return true;
411 }
412 if (check_flags(jedge, Topology::BC | Topology::EAST)) {
413 return true;
414 }
415 return false;
416 };
417 auto periodic_east = [&](idx_t jedge) {
418 if (check_flags(jedge, Topology::PERIODIC | Topology::EAST)) {
419 return true;
420 }
421 return false;
422 };
423 auto periodic_west = [&](idx_t jedge) {
424 if (check_flags(jedge, Topology::PERIODIC | Topology::WEST)) {
425 return true;
426 }
427 return false;
428 };
429
430 auto periodic_west_bdry = [&](idx_t jedge) {
431 if (check_flags(jedge, Topology::PERIODIC | Topology::WEST | Topology::BC)) {
432 return true;
433 }
434 return false;
435 };
436
437
438 idx_t nb_edges = edges.size();
439
440 std::vector<gidx_t> bdry_edges;
441 bdry_edges.reserve(nb_edges);
442 std::map<gidx_t, idx_t> global_to_local;
443
444
445 PeriodicTransform transform_periodic_east(-360.);
446 PeriodicTransform transform_periodic_west(+360.);
447 UniqueLonLat _compute_uid(mesh);
448 auto compute_uid = [&](idx_t jedge) -> gidx_t {
449 if (periodic_east(jedge)) {
450 return -_compute_uid(edge_nodes.row(jedge), transform_periodic_east);
451 }
452 else if (periodic_west_bdry(jedge)) {
453 return _compute_uid(edge_nodes.row(jedge));
454 }
455 else if (periodic_west(jedge)) {
456 return -_compute_uid(edge_nodes.row(jedge), transform_periodic_west);
457 }
458 else {
459 return _compute_uid(edge_nodes.row(jedge));
460 }
461 };
462
463
464 // should be unit-test
465 {
466 ATLAS_ASSERT(util::unique_lonlat(360., 0., transform_periodic_east) == util::unique_lonlat(0., 0.));
467 ATLAS_ASSERT(util::unique_lonlat(0., 0., transform_periodic_west) == util::unique_lonlat(360., 0.));
468 }
469
470
471 for (idx_t jedge = 0; jedge < nb_edges; ++jedge) {
472 gidx_t edge_gidx = compute_uid(jedge);
473 global_to_local[edge_gidx] = jedge;
474
475 #ifdef DEBUGGING_PARFIELDS
476 if (FIND_EDGE(jedge)) {
477 std::cout << "[" << mypart << "] " << EDGE(jedge) << " has gidx " << edge_gidx << std::endl;
478 if (periodic_east(jedge)) {
479 std::cout << "[" << mypart << "] " << EDGE(jedge) << " is periodic_east " << std::endl;
480 }
481 else if (periodic_west(jedge)) {
482 std::cout << "[" << mypart << "] " << EDGE(jedge) << " is periodic_west " << std::endl;
483 }
484 else {
485 std::cout << "[" << mypart << "] " << EDGE(jedge) << " is not periodic" << std::endl;
486 }
487 }
488 if (FIND_GIDX(edge_gidx))
489 std::cout << "[" << mypart << "] "
490 << "has " << EDGE(jedge) << " with gidx " << edge_gidx << std::endl;
491 #endif
492
493 idx_t ip1 = edge_nodes(jedge, 0);
494 idx_t ip2 = edge_nodes(jedge, 1);
495 int pn1 = node_part(ip1);
496 int pn2 = node_part(ip2);
497 int y1 = util::microdeg(xy(ip1, YY));
498 int y2 = util::microdeg(xy(ip2, YY));
499 int p;
500 if (y1 == y2) {
501 int x1 = util::microdeg(xy(ip1, XX));
502 int x2 = util::microdeg(xy(ip2, XX));
503 p = (x1 < x2) ? pn1 : pn2;
504 }
505 else {
506 p = (y1 > y2) ? pn1 : pn2;
507 }
508
509 idx_t elem1 = edge_to_elem(jedge, 0);
510 idx_t elem2 = edge_to_elem(jedge, 1);
511 idx_t missing = edge_to_elem.missing_value();
512 if (elem1 == missing && elem2 == missing) {
513 // Don't attempt to change p
514 }
515 else if (elem1 == missing) {
516 ATLAS_NOTIMPLEMENTED;
517 }
518 else if (elem2 == missing) {
519 if (pn1 == pn2) {
520 p = pn1;
521 }
522 else if (periodic_east(jedge)) {
523 #ifdef DEBUGGING_PARFIELDS
524 if (FIND_EDGE(jedge))
525 std::cout << "[" << mypart << "] "
526 << "periodic_east" << std::endl;
527 #endif
528 bdry_edges.push_back(edge_gidx);
529 p = -1;
530 }
531 else if (periodic_west_bdry(jedge)) {
532 #ifdef DEBUGGING_PARFIELDS
533 if (FIND_EDGE(jedge))
534 std::cout << "[" << mypart << "] "
535 << "periodic_west_bdry" << std::endl;
536 #endif
537 p = elem_part(elem1);
538 }
539 else if (periodic_west(jedge)) {
540 #ifdef DEBUGGING_PARFIELDS
541 if (FIND_EDGE(jedge))
542 std::cout << "[" << mypart << "] "
543 << "periodic_west" << std::endl;
544 #endif
545 bdry_edges.push_back(edge_gidx);
546 p = -1;
547 }
548 else if (domain_bdry(jedge)) {
549 #ifdef DEBUGGING_PARFIELDS
550 if (FIND_EDGE(jedge))
551 std::cout << "[" << mypart << "] "
552 << "domain_bdry" << std::endl;
553 #endif
554 p = elem_part(elem1);
555 }
556 else {
557 bdry_edges.push_back(edge_gidx);
558 p = -1;
559 }
560 }
561 else if (p != elem_part(elem1) && p != elem_part(elem2)) {
562 p = (p == pn1) ? pn2 : pn1;
563
564 if (p != elem_part(elem1) and p != elem_part(elem2)) {
565 std::stringstream msg;
566 msg << "[" << mpi::rank() << "] " << EDGE(jedge) << " has nodes and elements of different rank: elem1[p"
567 << elem_part(elem1) << "] elem2[p" << elem_part(elem2) << "]";
568 throw_Exception(msg.str(), Here());
569 }
570 }
571 edge_part(jedge) = p;
572 }
573 std::sort(bdry_edges.begin(), bdry_edges.end());
574 auto is_bdry_edge = [&bdry_edges](gidx_t gid) {
575 std::vector<uid_t>::iterator it = std::lower_bound(bdry_edges.begin(), bdry_edges.end(), gid);
576 bool found = !(it == bdry_edges.end() || gid < *it);
577 return found;
578 };
579
580 int mpi_size = mpi::size();
581 mpi::Buffer<gidx_t, 1> recv_bdry_edges_from_parts(mpi_size);
582 std::vector<std::vector<gidx_t>> send_gidx(mpi_size);
583 std::vector<std::vector<int>> send_part(mpi_size);
584 std::vector<std::vector<gidx_t>> send_bdry_gidx(mpi_size);
585 std::vector<std::vector<int>> send_bdry_elem_part(mpi_size);
586 std::vector<std::vector<gidx_t>> send_bdry_elem_gidx(mpi_size);
587 mpi::comm().allGatherv(bdry_edges.begin(), bdry_edges.end(), recv_bdry_edges_from_parts);
588 for (int p = 0; p < mpi_size; ++p) {
589 auto view = recv_bdry_edges_from_parts[p];
590 for (size_t j = 0; j < view.size(); ++j) {
591 gidx_t gidx = view[j];
592 gidx_t master_gidx = std::abs(gidx);
593 if (global_to_local.count(master_gidx)) {
594 idx_t iedge = global_to_local[master_gidx];
595 #ifdef DEBUGGING_PARFIELDS
596 if (FIND_GIDX(master_gidx))
597 std::cout << "[" << mypart << "] found " << EDGE(iedge) << std::endl;
598 #endif
599 if (not is_bdry_edge(master_gidx)) {
600 send_gidx[p].push_back(gidx);
601 send_part[p].push_back(edge_part(iedge));
602 #ifdef DEBUGGING_PARFIELDS
603 if (FIND_EDGE(iedge)) {
604 std::cout << "[" << mypart << "] found " << EDGE(iedge) " for part " << p << std::endl;
605 }
606 #endif
607 }
608 else { // boundary edge with nodes of different rank
609 idx_t ielem = (edge_to_elem(iedge, 0) != edge_to_elem.missing_value() ? edge_to_elem(iedge, 0)
610 : edge_to_elem(iedge, 1));
611 send_bdry_gidx[p].push_back(gidx);
612 send_bdry_elem_part[p].push_back(elem_part(ielem));
613 send_bdry_elem_gidx[p].push_back(elem_gidx(ielem));
614 }
615 }
616 }
617 }
618 std::vector<std::vector<gidx_t>> recv_gidx(mpi_size);
619 std::vector<std::vector<int>> recv_part(mpi_size);
620
621 mpi::comm().allToAll(send_gidx, recv_gidx);
622 mpi::comm().allToAll(send_part, recv_part);
623 for (int p = 0; p < mpi_size; ++p) {
624 const auto& recv_gidx_p = recv_gidx[p];
625 const auto& recv_part_p = recv_part[p];
626 for (size_t j = 0; j < recv_gidx_p.size(); ++j) {
627 idx_t iedge = global_to_local[recv_gidx_p[j]];
628 // int prev = edge_part( iedge );
629 edge_part(iedge) = recv_part_p[j];
630 // if( edge_part(iedge) != prev )
631 // Log::error() << EDGE(iedge) << " part " << prev << " --> " <<
632 // edge_part(iedge) << std::endl;
633 }
634 }
635
636 std::vector<std::vector<gidx_t>> recv_bdry_gidx(mpi_size);
637 std::vector<std::vector<int>> recv_bdry_elem_part(mpi_size);
638 std::vector<std::vector<gidx_t>> recv_bdry_elem_gidx(mpi_size);
639 mpi::comm().allToAll(send_bdry_gidx, recv_bdry_gidx);
640 mpi::comm().allToAll(send_bdry_elem_part, recv_bdry_elem_part);
641 mpi::comm().allToAll(send_bdry_elem_gidx, recv_bdry_elem_gidx);
642 for (int p = 0; p < mpi_size; ++p) {
643 const auto& recv_bdry_gidx_p = recv_bdry_gidx[p];
644 const auto& recv_bdry_elem_part_p = recv_bdry_elem_part[p];
645 const auto& recv_bdry_elem_gidx_p = recv_bdry_elem_gidx[p];
646 for (size_t j = 0; j < recv_bdry_gidx_p.size(); ++j) {
647 idx_t iedge = global_to_local[recv_bdry_gidx_p[j]];
648 idx_t e1 = (edge_to_elem(iedge, 0) != edge_to_elem.missing_value() ? edge_to_elem(iedge, 0)
649 : edge_to_elem(iedge, 1));
650 if (elem_gidx(e1) != recv_bdry_elem_gidx_p[j]) {
651 idx_t ip1 = edge_nodes(iedge, 0);
652 idx_t ip2 = edge_nodes(iedge, 1);
653 int pn1 = node_part(ip1);
654 int pn2 = node_part(ip2);
655 int y1 = util::microdeg(xy(ip1, YY));
656 int y2 = util::microdeg(xy(ip2, YY));
657 int ped;
658 if (y1 == y2) {
659 int x1 = util::microdeg(xy(ip1, XX));
660 int x2 = util::microdeg(xy(ip2, XX));
661 ped = (x1 < x2) ? pn1 : pn2;
662 }
663 else {
664 ped = (y1 > y2) ? pn1 : pn2;
665 }
666 int pe1 = elem_part(e1);
667 int pe2 = recv_bdry_elem_part_p[j];
668 if (ped != pe1 && ped != pe2) {
669 ped = (ped == pn1) ? pn2 : pn1;
670 if (ped != pe1 && p != pe2) {
671 std::stringstream msg;
672 msg << "[" << mpi::rank() << "] " << EDGE(iedge)
673 << " has nodes and elements of different rank: elem1[p" << pe1 << "] elem2[p" << pe2 << "]";
674 throw_Exception(msg.str(), Here());
675 }
676 }
677 edge_part(iedge) = ped;
678 }
679 }
680 }
681
682 // Sanity check
683 auto edge_flags = array::make_view<int, 1>(edges.flags());
684 auto is_pole_edge = [&](idx_t e) { return Topology::check(edge_flags(e), Topology::POLE); };
685 bool has_pole_edges = false;
686 mesh.edges().metadata().get("pole_edges", has_pole_edges);
687 int insane = 0;
688 for (idx_t jedge = 0; jedge < nb_edges; ++jedge) {
689 idx_t ip1 = edge_nodes(jedge, 0);
690 idx_t ip2 = edge_nodes(jedge, 1);
691 idx_t elem1 = edge_to_elem(jedge, 0);
692 idx_t elem2 = edge_to_elem(jedge, 1);
693 int p = edge_part(jedge);
694 int pn1 = node_part(ip1);
695 int pn2 = node_part(ip2);
696 if (has_pole_edges && is_pole_edge(jedge)) {
697 if (p != pn1 || p != pn2) {
698 Log::error() << "pole edge " << EDGE(jedge) << " [p" << p << "] is not correct" << std::endl;
699 insane = 1;
700 }
701 }
702 else {
703 if (elem1 == edge_to_elem.missing_value() && elem2 == edge_to_elem.missing_value()) {
704 Log::error() << EDGE(jedge) << " has no neighbouring elements" << std::endl;
705 insane = 1;
706 }
707 }
708 bool edge_is_partition_boundary =
709 (elem1 == edge_to_elem.missing_value() || elem2 == edge_to_elem.missing_value());
710 bool edge_partition_is_same_as_one_of_nodes = (p == pn1 || p == pn2);
711 if (edge_is_partition_boundary) {
712 if (not edge_partition_is_same_as_one_of_nodes) {
713 gidx_t edge_gidx = compute_uid(jedge);
714
715 if (elem1 != edge_to_elem.missing_value()) {
716 Log::error() << "[" << mypart << "] " << EDGE(jedge) << " " << edge_gidx << " [p" << p
717 << "] at partition_boundary is not correct. elem1[p" << elem_part(elem1) << "]"
718 << std::endl;
719 }
720 else {
721 Log::error() << "[" << mypart << "] " << EDGE(jedge) << " " << edge_gidx << " [p" << p
722 << "] at partition_boundary is not correct elem2[p" << elem_part(elem2) << "]"
723 << std::endl;
724 }
725 insane = 1;
726 }
727 }
728 else {
729 int pe1 = elem_part(elem1);
730 int pe2 = elem_part(elem2);
731 bool edge_partition_is_same_as_one_of_elems = (p == pe1 || p == pe2);
732 if (not edge_partition_is_same_as_one_of_elems and not edge_partition_is_same_as_one_of_nodes) {
733 Log::error() << EDGE(jedge) << " is not correct elem1[p" << pe1 << "] elem2[p" << pe2 << "]"
734 << std::endl;
735 insane = 1;
736 }
737 }
738 }
739 mpi::comm().allReduceInPlace(insane, eckit::mpi::max());
740 if (insane && mpi::rank() == 0) {
741 throw_Exception("Sanity check failed", Here());
742 }
743
744 //#ifdef DEBUGGING_PARFIELDS
745 // if( OWNED_EDGE(jedge) )
746 // DEBUG( EDGE(jedge) << " --> part " << edge_part(jedge));
747 //#endif
748
749 //#ifdef DEBUGGING_PARFIELDS_disable
750 // if( PERIODIC_EDGE(jedge) )
751 // DEBUG_VAR( " the part is " << edge_part(jedge) );
752 //#endif
753 // }
754
755 return edges.partition();
756 }
757
758 Field& build_edges_remote_idx(Mesh& mesh) {
759 ATLAS_TRACE();
760
761 const mesh::Nodes& nodes = mesh.nodes();
762 UniqueLonLat compute_uid(mesh);
763
764 idx_t mypart = mpi::rank();
765 idx_t nparts = mpi::size();
766
767 mesh::HybridElements& edges = mesh.edges();
768
769 auto edge_ridx = array::make_indexview<idx_t, 1>(edges.remote_index());
770
771 const array::ArrayView<int, 1> edge_part = array::make_view<int, 1>(edges.partition());
772 const mesh::HybridElements::Connectivity& edge_nodes = edges.node_connectivity();
773
774 array::ArrayView<const double, 2> xy = array::make_view<double, 2>(nodes.xy());
775 array::ArrayView<const int, 1> flags = array::make_view<int, 1>(nodes.flags());
776 #ifdef DEBUGGING_PARFIELDS
777 array::ArrayView<gidx_t, 1> node_gidx = array::make_view<gidx_t, 1>(nodes.global_index());
778 array::ArrayView<int, 1> node_part = array::make_view<int, 1>(nodes.partition());
779 #endif
780
781 auto edge_flags = array::make_view<int, 1>(edges.flags());
782 auto is_pole_edge = [&](idx_t e) { return Topology::check(edge_flags(e), Topology::POLE); };
783 bool has_pole_edges = false;
784 mesh.edges().metadata().get("pole_edges", has_pole_edges);
785
786 const int nb_edges = edges.size();
787
788 double centroid[2];
789 std::vector<std::vector<uid_t>> send_needed(mpi::size());
790 std::vector<std::vector<uid_t>> recv_needed(mpi::size());
791 std::map<uid_t, int> lookup;
792
793 PeriodicTransform transform;
794
795 for (int jedge = 0; jedge < nb_edges; ++jedge) {
796 int ip1 = edge_nodes(jedge, 0);
797 int ip2 = edge_nodes(jedge, 1);
798 centroid[XX] = 0.5 * (xy(ip1, XX) + xy(ip2, XX));
799 centroid[YY] = 0.5 * (xy(ip1, YY) + xy(ip2, YY));
800 if (has_pole_edges && is_pole_edge(jedge)) {
801 centroid[YY] = centroid[YY] > 0 ? 90. : -90.;
802 }
803
804 bool needed(false);
805
806 if ((Topology::check(flags(ip1), Topology::PERIODIC) &&
807 !Topology::check(flags(ip1), Topology::BC | Topology::WEST) &&
808 Topology::check(flags(ip2), Topology::PERIODIC) &&
809 !Topology::check(flags(ip2), Topology::BC | Topology::WEST)) ||
810 (Topology::check(flags(ip1), Topology::PERIODIC) &&
811 !Topology::check(flags(ip1), Topology::BC | Topology::WEST) &&
812 Topology::check(flags(ip2), Topology::BC | Topology::WEST)) ||
813 (Topology::check(flags(ip1), Topology::BC | Topology::WEST) &&
814 Topology::check(flags(ip2), Topology::PERIODIC) &&
815 !Topology::check(flags(ip2), Topology::BC | Topology::WEST))) {
816 needed = true;
817 if (Topology::check(flags(ip1), Topology::EAST)) {
818 transform(centroid, -1);
819 }
820 else {
821 transform(centroid, +1);
822 }
823 }
824
825 uid_t uid = util::unique_lonlat(centroid);
826 if (idx_t(edge_part(jedge)) == mypart && !needed) // All interior edges fall here
827 {
828 lookup[uid] = jedge;
829 edge_ridx(jedge) = jedge;
830
831 #ifdef DEBUGGING_PARFIELDS
832 if (FIND_EDGE(jedge)) {
833 ATLAS_DEBUG("Found " << EDGE(jedge));
834 }
835 #endif
836 }
837 else // All ghost edges PLUS the periodic edges identified edges above
838 // fall here
839 {
840 send_needed[edge_part(jedge)].push_back(uid);
841 send_needed[edge_part(jedge)].push_back(jedge);
842 #ifdef DEBUGGING_PARFIELDS
843 send_needed[edge_part(jedge)].push_back(node_gidx(ip1));
844 send_needed[edge_part(jedge)].push_back(node_gidx(ip2));
845 send_needed[edge_part(jedge)].push_back(node_part(ip1));
846 send_needed[edge_part(jedge)].push_back(node_part(ip2));
847 #endif
848 }
849 }
850
851 idx_t varsize = 2;
852 #ifdef DEBUGGING_PARFIELDS
853 varsize = 6;
854 #endif
855
856 ATLAS_TRACE_MPI(ALLTOALL) { mpi::comm().allToAll(send_needed, recv_needed); }
857
858 std::vector<std::vector<int>> send_found(mpi::size());
859 std::vector<std::vector<int>> recv_found(mpi::size());
860
861 std::map<uid_t, int>::iterator found;
862 for (idx_t jpart = 0; jpart < nparts; ++jpart) {
863 const std::vector<uid_t>& recv_edge = recv_needed[jpart];
864 const idx_t nb_recv_edges = idx_t(recv_edge.size()) / varsize;
865 // array::ArrayView<uid_t,2> recv_edge( recv_needed[ jpart ].data(),
866 // array::make_shape(recv_needed[ jpart ].size()/varsize,varsize) );
867 for (idx_t jedge = 0; jedge < nb_recv_edges; ++jedge) {
868 uid_t recv_uid = recv_edge[jedge * varsize + 0];
869 int recv_idx = recv_edge[jedge * varsize + 1];
870 found = lookup.find(recv_uid);
871 if (found != lookup.end()) {
872 send_found[jpart].push_back(recv_idx);
873 send_found[jpart].push_back(found->second);
874 }
875 else {
876 std::stringstream msg;
877 #ifdef DEBUGGING_PARFIELDS
878 msg << "Edge(" << recv_edge[jedge * varsize + 2] << "[p" << recv_edge[jedge * varsize + 4] << "] "
879 << recv_edge[jedge * varsize + 3] << "[p" << recv_edge[jedge * varsize + 5] << "])";
880 #else
881 msg << "Edge with uid " << recv_uid;
882 #endif
883 msg << " requested by rank [" << jpart << "]";
884 msg << " that should be owned by " << mpi::rank()
885 << " is not found. This could be because no "
886 "halo was built.";
887 // throw_Exception(msg.str(),Here());
888 Log::warning() << msg.str() << " @ " << Here() << std::endl;
889 }
890 }
891 }
892
893 ATLAS_TRACE_MPI(ALLTOALL) { mpi::comm().allToAll(send_found, recv_found); }
894
895 for (idx_t jpart = 0; jpart < nparts; ++jpart) {
896 const std::vector<int>& recv_edge = recv_found[jpart];
897 const idx_t nb_recv_edges = recv_edge.size() / 2;
898 // array::ArrayView<int,2> recv_edge( recv_found[ jpart ].data(),
899 // array::make_shape(recv_found[ jpart ].size()/2,2) );
900 for (idx_t jedge = 0; jedge < nb_recv_edges; ++jedge) {
901 edge_ridx(recv_edge[jedge * 2 + 0]) = recv_edge[jedge * 2 + 1];
902 }
903 }
904 return edges.remote_index();
905 }
906
907 Field& build_edges_global_idx(Mesh& mesh) {
908 ATLAS_TRACE();
909
910 UniqueLonLat compute_uid(mesh);
911
912 int nparts = mpi::size();
913 idx_t root = 0;
914
915 mesh::HybridElements& edges = mesh.edges();
916
917 array::make_view<gidx_t, 1>(edges.global_index()).assign(-1);
918 array::ArrayView<gidx_t, 1> edge_gidx = array::make_view<gidx_t, 1>(edges.global_index());
919
920 const mesh::HybridElements::Connectivity& edge_nodes = edges.node_connectivity();
921 array::ArrayView<double, 2> xy = array::make_view<double, 2>(mesh.nodes().xy());
922 auto edge_flags = array::make_view<int, 1>(edges.flags());
923 auto is_pole_edge = [&](idx_t e) { return Topology::check(edge_flags(e), Topology::POLE); };
924 bool has_pole_edges = false;
925 mesh.edges().metadata().get("pole_edges", has_pole_edges);
926
927 /*
928 * Sorting following edge_gidx will define global order of
929 * gathered fields. Special care needs to be taken for
930 * pole edges, as their centroid might coincide with
931 * other edges
932 */
933 double centroid[2];
934 int nb_edges = edges.size();
935 for (int jedge = 0; jedge < nb_edges; ++jedge) {
936 if (edge_gidx(jedge) <= 0) {
937 centroid[XX] = 0.5 * (xy(edge_nodes(jedge, 0), XX) + xy(edge_nodes(jedge, 1), XX));
938 centroid[YY] = 0.5 * (xy(edge_nodes(jedge, 0), YY) + xy(edge_nodes(jedge, 1), YY));
939 if (has_pole_edges && is_pole_edge(jedge)) {
940 centroid[YY] = centroid[YY] > 0 ? 90. : -90.;
941 }
942 edge_gidx(jedge) = util::unique_lonlat(centroid);
943 }
944 }
945
946 /*
947 * REMOTE INDEX BASE = 1
948 */
949
950 // 1) Gather all global indices, together with location
951 array::ArrayT<uid_t> loc_edge_id_arr(nb_edges);
952 array::ArrayView<uid_t, 1> loc_edge_id = array::make_view<uid_t, 1>(loc_edge_id_arr);
953
954 for (int jedge = 0; jedge < nb_edges; ++jedge) {
955 loc_edge_id(jedge) = edge_gidx(jedge);
956 }
957
958 std::vector<int> recvcounts(mpi::size());
959 std::vector<int> recvdispls(mpi::size());
960
961 ATLAS_TRACE_MPI(GATHER) { mpi::comm().gather(nb_edges, recvcounts, root); }
962
963 recvdispls[0] = 0;
964 for (int jpart = 1; jpart < nparts; ++jpart) // start at 1
965 {
966 recvdispls[jpart] = recvcounts[jpart - 1] + recvdispls[jpart - 1];
967 }
968 int glb_nb_edges = std::accumulate(recvcounts.begin(), recvcounts.end(), 0);
969
970 array::ArrayT<uid_t> glb_edge_id_arr(glb_nb_edges);
971 array::ArrayView<uid_t, 1> glb_edge_id = array::make_view<uid_t, 1>(glb_edge_id_arr);
972
973 ATLAS_TRACE_MPI(GATHER) {
974 mpi::comm().gatherv(loc_edge_id.data(), loc_edge_id.size(), glb_edge_id.data(), recvcounts.data(),
975 recvdispls.data(), root);
976 }
977
978 // 2) Sort all global indices, and renumber from 1 to glb_nb_edges
979 std::vector<Node> edge_sort;
980 edge_sort.reserve(glb_nb_edges);
981 for (idx_t jedge = 0; jedge < glb_edge_id.shape(0); ++jedge) {
982 edge_sort.emplace_back(glb_edge_id(jedge), jedge);
983 }
984 std::sort(edge_sort.begin(), edge_sort.end());
985
986 // Assume edge gid start
987 uid_t gid(0);
988 for (size_t jedge = 0; jedge < edge_sort.size(); ++jedge) {
989 if (jedge == 0) {
990 ++gid;
991 }
992 else if (edge_sort[jedge].g != edge_sort[jedge - 1].g) {
993 ++gid;
994 }
995 int iedge = edge_sort[jedge].i;
996 glb_edge_id(iedge) = gid;
997 }
998
999 // 3) Scatter renumbered back
1000 ATLAS_TRACE_MPI(SCATTER) {
1001 mpi::comm().scatterv(glb_edge_id.data(), recvcounts.data(), recvdispls.data(), loc_edge_id.data(),
1002 loc_edge_id.size(), root);
1003 }
1004
1005 for (int jedge = 0; jedge < nb_edges; ++jedge) {
1006 edge_gidx(jedge) = loc_edge_id(jedge);
1007 }
1008
1009 return edges.global_index();
1010 }
1011
1012 //----------------------------------------------------------------------------------------------------------------------
1013
1014 Field& build_cells_remote_idx(mesh::Cells& cells, const mesh::Nodes& nodes) {
1015 ATLAS_TRACE();
1016 idx_t mypart = static_cast<idx_t>(mpi::rank());
1017 idx_t nparts = static_cast<idx_t>(mpi::size());
1018
1019 // This piece should be somewhere central ... could be NPROMA ?
1020 // ---------->
1021 std::vector<idx_t> proc(nparts);
1022 for (idx_t jpart = 0; jpart < nparts; ++jpart) {
1023 proc[jpart] = jpart;
1024 }
1025 // <---------
1026
1027 auto ridx = array::make_indexview<idx_t, 1>(cells.remote_index());
1028 const auto part = array::make_view<int, 1>(cells.partition());
1029 const auto gidx = array::make_view<gidx_t, 1>(cells.global_index());
1030 const auto flags = array::make_view<int, 1>(cells.flags());
1031 const auto& element_nodes = cells.node_connectivity();
1032 idx_t nb_cells = cells.size();
1033
1034 const PeriodicTransform transform_periodic_east(-360.);
1035 const PeriodicTransform transform_periodic_west(+360.);
1036 const UniqueLonLat compute_uid_lonlat(nodes);
1037 auto compute_uid = [&](idx_t jcell) {
1038 constexpr int PERIODIC = util::Topology::PERIODIC;
1039 constexpr int EAST = util::Topology::EAST;
1040 constexpr int WEST = util::Topology::WEST;
1041 const auto flags_view = util::Bitflags::view(flags(jcell));
1042 if (flags_view.check(PERIODIC | EAST)) {
1043 return compute_uid_lonlat(element_nodes.row(jcell), transform_periodic_east);
1044 }
1045 if (flags_view.check(PERIODIC | WEST)) {
1046 return compute_uid_lonlat(element_nodes.row(jcell), transform_periodic_west);
1047 }
1048 return compute_uid_lonlat(element_nodes.row(jcell));
1049 };
1050
1051 idx_t varsize = 2;
1052
1053 std::vector<std::vector<uid_t>> send_needed(mpi::size());
1054 std::vector<std::vector<uid_t>> recv_needed(mpi::size());
1055 std::map<uid_t, int> lookup;
1056 for (idx_t jcell = 0; jcell < nb_cells; ++jcell) {
1057 uid_t uid = compute_uid(jcell);
1058
1059 if (idx_t(part(jcell)) == mypart) {
1060 lookup[uid] = jcell;
1061 ridx(jcell) = jcell;
1062 }
1063 else {
1064 ATLAS_ASSERT(jcell < part.shape(0));
1065 if (part(jcell) >= static_cast<int>(proc.size())) {
1066 std::stringstream msg;
1067 msg << "Assertion [part(" << jcell << ") < proc.size()] failed\n"
1068 << "part(" << jcell << ") = " << part(jcell) << "\n"
1069 << "proc.size() = " << proc.size();
1070 throw_AssertionFailed(msg.str(), Here());
1071 }
1072 ATLAS_ASSERT(part(jcell) < (idx_t)proc.size());
1073 ATLAS_ASSERT((size_t)proc[part(jcell)] < send_needed.size());
1074 send_needed[proc[part(jcell)]].push_back(uid);
1075 send_needed[proc[part(jcell)]].push_back(jcell);
1076 }
1077 }
1078
1079 ATLAS_TRACE_MPI(ALLTOALL) { mpi::comm().allToAll(send_needed, recv_needed); }
1080
1081 std::vector<std::vector<int>> send_found(mpi::size());
1082 std::vector<std::vector<int>> recv_found(mpi::size());
1083
1084 for (idx_t jpart = 0; jpart < nparts; ++jpart) {
1085 const std::vector<uid_t>& recv_cell = recv_needed[proc[jpart]];
1086 const idx_t nb_recv_cells = idx_t(recv_cell.size()) / varsize;
1087 for (idx_t jcell = 0; jcell < nb_recv_cells; ++jcell) {
1088 uid_t uid = recv_cell[jcell * varsize + 0];
1089 int icell = recv_cell[jcell * varsize + 1];
1090 send_found[proc[jpart]].push_back(icell);
1091 send_found[proc[jpart]].push_back(lookup.count(uid) ? lookup[uid] : -1);
1092 }
1093 }
1094
1095 ATLAS_TRACE_MPI(ALLTOALL) { mpi::comm().allToAll(send_found, recv_found); }
1096
1097 std::stringstream errstream;
1098 size_t failed{0};
1099 for (idx_t jpart = 0; jpart < nparts; ++jpart) {
1100 const std::vector<int>& recv_cell = recv_found[proc[jpart]];
1101 const idx_t nb_recv_cells = recv_cell.size() / 2;
1102 // array::ArrayView<int,2> recv_node( recv_found[ proc[jpart] ].data(),
1103 // array::make_shape(recv_found[ proc[jpart] ].size()/2,2) );
1104 for (idx_t jcell = 0; jcell < nb_recv_cells; ++jcell) {
1105 idx_t icell = recv_cell[jcell * 2 + 0];
1106 idx_t ridx_icell = recv_cell[jcell * 2 + 1];
1107 if (ridx_icell >= 0) {
1108 ridx(icell) = ridx_icell;
1109 }
1110 else {
1111 ++failed;
1112 errstream << "\n[" << mpi::rank() << "] "
1113 << "Cell " << gidx(icell) << " not found on part [" << part(icell) << "]";
1114 }
1115 }
1116 }
1117 if (failed) {
1118 throw_AssertionFailed(errstream.str(), Here());
1119 }
1120 return cells.remote_index();
1121 }
1122
1123 void build_cells_parallel_fields(Mesh& mesh) {
1124 mpi::Scope mpi_scope(mesh.mpi_comm());
1125
1126 bool parallel = false;
1127 mesh.cells().metadata().get("parallel", parallel);
1128 if (!parallel) {
1129 build_cells_remote_idx(mesh.cells(), mesh.nodes());
1130 }
1131
1132 mesh.cells().metadata().set("parallel", true);
1133 }
1134
1135 //----------------------------------------------------------------------------------------------------------------------
1136 // C wrapper interfaces to C++ routines
1137
1138 void atlas__build_parallel_fields(Mesh::Implementation* mesh) {
1139 ATLAS_ASSERT(mesh != nullptr, "Cannot access uninitialised atlas_Mesh");
1140 Mesh m(mesh);
1141 build_parallel_fields(m);
1142 }
1143 void atlas__build_nodes_parallel_fields(mesh::Nodes* nodes) {
1144 ATLAS_ASSERT(nodes != nullptr, "Cannot access uninitialised atlas_mesh_Nodes");
1145 build_nodes_parallel_fields(*nodes);
1146 }
1147
1148 void atlas__build_edges_parallel_fields(Mesh::Implementation* mesh) {
1149 ATLAS_ASSERT(mesh != nullptr, "Cannot access uninitialised atlas_Mesh");
1150 Mesh m(mesh);
1151 build_edges_parallel_fields(m);
1152 }
1153
1154 void atlas__renumber_nodes_glb_idx(mesh::Nodes* nodes) {
1155 ATLAS_ASSERT(nodes != nullptr, "Cannot access uninitialised atlas_mesh_Nodes");
1156 renumber_nodes_glb_idx(*nodes);
1157 }
1158
1159 //----------------------------------------------------------------------------------------------------------------------
1160
1161 } // namespace actions
1162 } // namespace mesh
1163 } // namespace atlas