| 1 |
/* |
| 2 |
* (C) Copyright 2013 ECMWF. |
| 3 |
* |
| 4 |
* This software is licensed under the terms of the Apache Licence Version 2.0 |
| 5 |
* which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. |
| 6 |
* In applying this licence, ECMWF does not waive the privileges and immunities |
| 7 |
* granted to it by virtue of its status as an intergovernmental organisation |
| 8 |
* nor does it submit to any jurisdiction. |
| 9 |
*/ |
| 10 |
|
| 11 |
#include "atlas/grid/detail/partitioner/MatchingFunctionSpacePartitionerLonLatPolygon.h" |
| 12 |
|
| 13 |
#include <vector> |
| 14 |
|
| 15 |
#include "eckit/config/Resource.h" |
| 16 |
#include "eckit/log/ProgressTimer.h" |
| 17 |
|
| 18 |
#include "atlas/grid/Grid.h" |
| 19 |
#include "atlas/grid/Iterator.h" |
| 20 |
#include "atlas/parallel/mpi/mpi.h" |
| 21 |
#include "atlas/runtime/Exception.h" |
| 22 |
#include "atlas/runtime/Log.h" |
| 23 |
#include "atlas/util/CoordinateEnums.h" |
| 24 |
#include "atlas/util/PolygonXY.h" |
| 25 |
|
| 26 |
#include "atlas/parallel/omp/fill.h" |
| 27 |
#include "atlas/parallel/omp/omp.h" |
| 28 |
|
| 29 |
namespace atlas { |
| 30 |
namespace grid { |
| 31 |
namespace detail { |
| 32 |
namespace partitioner { |
| 33 |
|
| 34 |
namespace { |
| 35 |
// We did not yet implement self-registration in MatchedPartitionerFactory |
| 36 |
// PartitionerBuilder<MatchingFunctionSpacePartitionerLonLatPolygon> __builder( "lonlat-polygon" ); |
| 37 |
} |
| 38 |
|
| 39 |
void MatchingFunctionSpacePartitionerLonLatPolygon::partition(const Grid& grid, int part[]) const { |
| 40 |
ATLAS_TRACE("MatchingFunctionSpacePartitionerLonLatPolygon"); |
| 41 |
//atlas::vector<int> part( grid.size() ); |
| 42 |
|
| 43 |
const auto& comm = mpi::comm(partitioned_.mpi_comm()); |
| 44 |
const int mpi_rank = int(comm.rank()); |
| 45 |
const int mpi_size = int(comm.size()); |
| 46 |
|
| 47 |
if (mpi_size == 1) { |
| 48 |
// shortcut |
| 49 |
omp::fill(part, part + grid.size(), 0); |
| 50 |
} |
| 51 |
else { |
| 52 |
const auto& poly = partitioned_.polygon(); |
| 53 |
const auto& proj = partitioned_.projection(); |
| 54 |
|
| 55 |
util::PolygonXY poly_xy{poly}; |
| 56 |
{ |
| 57 |
ATLAS_TRACE("point-in-polygon check for entire grid (" + std::to_string(grid.size()) + " points)"); |
| 58 |
size_t num_threads = atlas_omp_get_max_threads(); |
| 59 |
size_t chunk_size = grid.size() / (1000 * num_threads); |
| 60 |
if (chunk_size == 0) { |
| 61 |
chunk_size = grid.size(); |
| 62 |
} |
| 63 |
size_t chunks = num_threads == 1 ? 1 : std::max(size_t(1), size_t(grid.size()) / chunk_size); |
| 64 |
atlas_omp_pragma(omp parallel for schedule(dynamic,1)) |
| 65 |
for( size_t chunk=0; chunk < chunks; ++chunk) { |
| 66 |
const size_t begin = chunk * size_t(grid.size()) / chunks; |
| 67 |
const size_t end = (chunk + 1) * size_t(grid.size()) / chunks; |
| 68 |
auto lonlat_it = grid.lonlat().begin() + chunk * grid.size() / chunks; |
| 69 |
PointXY xy; |
| 70 |
for (size_t n = begin; n < end; ++n) { |
| 71 |
xy = proj.xy(*lonlat_it); |
| 72 |
if (poly_xy.contains(xy)) { |
| 73 |
part[n] = mpi_rank; |
| 74 |
} |
| 75 |
else { |
| 76 |
part[n] = -1; |
| 77 |
} |
| 78 |
++lonlat_it; |
| 79 |
} |
| 80 |
} |
| 81 |
} |
| 82 |
ATLAS_TRACE_MPI(ALLREDUCE) { comm.allReduceInPlace(part, grid.size(), eckit::mpi::max()); } |
| 83 |
} |
| 84 |
} |
| 85 |
|
| 86 |
|
| 87 |
#if 0 |
| 88 |
const auto& comm = mpi::comm(partitioned_.mpi_comm()); |
| 89 |
const int mpi_rank = int(comm.rank()); |
| 90 |
const int mpi_size = int(comm.size()); |
| 91 |
|
| 92 |
ATLAS_TRACE( "MatchingFunctionSpacePartitionerLonLatPolygon::partition" ); |
| 93 |
|
| 94 |
ATLAS_ASSERT( grid.domain().global() ); |
| 95 |
|
| 96 |
Log::debug() << "MatchingFunctionSpacePartitionerLonLatPolygon::partition" << std::endl; |
| 97 |
|
| 98 |
// FIXME: THIS IS A HACK! the coordinates include North/South Pole (first/last |
| 99 |
// partitions only) |
| 100 |
bool includesNorthPole = ( mpi_rank == 0 ); |
| 101 |
bool includesSouthPole = ( mpi_rank == mpi_size - 1 ); |
| 102 |
|
| 103 |
const util::PolygonXY poly{prePartitionedFunctionSpace_.polygon( 0 )}; |
| 104 |
|
| 105 |
{ |
| 106 |
eckit::ProgressTimer timer( "Partitioning", grid.size(), "point", double( 10 ), atlas::Log::trace() ); |
| 107 |
size_t i = 0; |
| 108 |
|
| 109 |
for ( const PointLonLat& P : grid.lonlat() ) { |
| 110 |
++timer; |
| 111 |
const bool atThePole = ( includesNorthPole && P[LAT] >= poly.coordinatesMax()[LAT] ) || |
| 112 |
( includesSouthPole && P[LAT] < poly.coordinatesMin()[LAT] ); |
| 113 |
|
| 114 |
partitioning[i++] = atThePole || poly.contains( P ) ? mpi_rank : -1; |
| 115 |
} |
| 116 |
} |
| 117 |
|
| 118 |
// Synchronize partitioning, do a sanity check |
| 119 |
comm.allReduceInPlace( partitioning, grid.size(), eckit::mpi::Operation::MAX ); |
| 120 |
const int min = *std::min_element( partitioning, partitioning + grid.size() ); |
| 121 |
if ( min < 0 ) { |
| 122 |
throw_Exception( |
| 123 |
"Could not find partition for target node (source " |
| 124 |
"FunctionSpace does not contain all target grid points)", |
| 125 |
Here() ); |
| 126 |
} |
| 127 |
#endif |
| 128 |
|
| 129 |
} // namespace partitioner |
| 130 |
} // namespace detail |
| 131 |
} // namespace grid |
| 132 |
} // namespace atlas |