Debian Codegraph @

src/atlas/util/GridPointsJSONWriter.cc

raw
1 /*
2 * (C) Copyright 2023 ECMWF.
3 *
4 * This software is licensed under the terms of the Apache Licence Version 2.0
5 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
6 * In applying this licence, ECMWF does not waive the privileges and immunities
7 * granted to it by virtue of its status as an intergovernmental organisation
8 * nor does it submit to any jurisdiction.
9 */
10
11 #include "GridPointsJSONWriter.h"
12
13 #include <iostream>
14 #include <iomanip>
15 #include <vector>
16 #include <string>
17
18 #include "eckit/utils/Tokenizer.h"
19
20 namespace atlas {
21 namespace util {
22
23 //------------------------------------------------------------------------------
24
25 namespace {
26 std::vector<long> points_from_list(const std::string& list, long base) {
27 std::vector<long> points;
28 if (list.empty()) {
29 return points;
30 }
31 if (list[0] == '[' && list[list.size()-1] == ']') {
32 return points_from_list(std::string(list.begin()+1,list.begin()+list.size()-1), base);
33 }
34 std::vector<std::string> points_ranges;
35 eckit::Tokenizer{","}(list,points_ranges);
36 auto tokenize_range = eckit::Tokenizer{"-"};
37 auto to_int = [](const std::string& s, const eckit::CodeLocation& here) -> long {
38 try {
39 return std::stol(s);
40 }
41 catch( std::exception& e) {
42 throw_Exception("Could not convert '" + s + "' to integer",here);
43 }
44 };
45 for (const auto& point_range: points_ranges) {
46 std::vector<std::string> point_range_split;
47 tokenize_range(point_range,point_range_split);
48 if (point_range_split.size() == 1) {
49 points.emplace_back(to_int(point_range,Here())-base);
50 }
51 else {
52 for (long p = to_int(point_range_split[0],Here()); p <= to_int(point_range_split[1],Here()); ++p) {
53 points.emplace_back(p-base);
54 }
55 }
56 }
57 return points;
58 }
59 }
60
61 //------------------------------------------------------------------------------
62
63 GridPointsJSONWriter::GridPointsJSONWriter(Grid grid, const eckit::Parametrisation& args) : grid_{grid} {
64 args.get("json.precision",precision_=-1);
65 args.get("verbose",verbose_=0);
66 if (not args.get("partitions",nb_partitions_=0)) {
67 args.get("partitioner.partitions",nb_partitions_);
68 }
69 if (not args.get("partition",partition_=-1)) {
70 args.get("partition",partition_);
71 }
72 args.get("json.pretty", pretty_=false);
73 args.get("field",field_="lonlat");
74 args.get("field_base",field_base_=0);
75 std::string points_list;
76 if (args.get("index",points_list)) {
77 args.get("index_base",points_base_ = 0);
78 points_ = points_from_list(points_list,points_base_);
79 }
80
81 if( nb_partitions_ > 0 ) {
82 auto partitioner_config = grid_.partitioner(); // recommended by the grid itself
83 std::string partitioner;
84 if (args.get("partitioner.type",partitioner)) {
85 partitioner_config.set("type", partitioner);
86 }
87 partitioner_config.set("partitions", nb_partitions_);
88 distribution_ = grid::Distribution{grid_,partitioner_config};
89 }
90 }
91
92 //------------------------------------------------------------------------------
93
94 void GridPointsJSONWriter::write(std::ostream& out, eckit::Channel& info) const {
95 write(out, &info);
96 }
97
98 //------------------------------------------------------------------------------
99
100 void GridPointsJSONWriter::write(std::ostream& out, std::ostream* info) const {
101
102 if (field_ == "none") {
103 return;
104 }
105
106 int points_newline = pretty_ ? true : false;
107 int points_indent = pretty_ ? 2 : 0;
108 int partition_indent = 0;
109 size_t chunk_size = 1000000;
110
111 auto saved_fmtflags = out.flags();
112 if (precision_ >= 0) {
113 out << std::fixed << std::setprecision(precision_);
114 }
115
116 auto writePoint = [&out](const Point2& p) {
117 out << "[" << p[0] << "," << p[1] << "]";
118 };
119
120 if(nb_partitions_ == 0 || field_ == "partition") {
121 out << "[";
122 if (pretty_) {
123 out << '\n';
124 }
125 std::string join = points_newline ? ",\n" : ",";
126 std::string indent(points_indent,' ');
127
128 size_t n{0};
129 if (points_.size()) {
130 if (field_ == "lonlat") {
131 auto lonlat = grid_.lonlat();
132 for (auto p: points_) {
133 if (p < grid_.size()) {
134 if (n!=0) {
135 out << join;
136 }
137 out << indent;
138 auto it = lonlat.begin()+(p);
139 writePoint(*it);
140 ++n;
141 }
142 }
143 }
144 else if (field_ == "index") {
145 for (auto p: points_) {
146 if (p < grid_.size()) {
147 if (n!=0) {
148 out << join;
149 }
150 out << indent;
151 out << p + field_base_;
152 ++n;
153 }
154 }
155 }
156 else if (field_ == "partition") {
157 for (auto p: points_) {
158 if (p < grid_.size()) {
159 if (n!=0) {
160 out << join;
161 }
162 out << indent;
163 out << distribution_.partition(p) + field_base_;
164 ++n;
165 }
166 }
167 }
168 else {
169 ATLAS_THROW_EXCEPTION("Cannot output field \""<<field_<<"\"");
170 }
171 }
172 else {
173 size_t end = 0;
174 size_t begin;
175 auto write_progress = [&]() {
176 if (info && verbose_ && grid_.size() > chunk_size) {
177 *info << std::fixed << std::setprecision(1) << double(begin)/double(grid_.size()) * 100 << "% completed" << std::endl;
178 }
179 };
180 auto write_progress_end = [&]() {
181 if (info && verbose_ && grid_.size() > chunk_size) {
182 *info << "100% completed" << std::endl;
183 }
184 };
185 while(end != grid_.size()) {
186 begin = end;
187 end = std::min<size_t>(grid_.size(),begin+chunk_size);
188 write_progress();
189 if (field_ == "lonlat") {
190 auto it = grid_.lonlat().begin()+begin;
191 for (size_t n=begin; n<end; ++n, ++it) {
192 if (n!=0) {
193 out << join;
194 }
195 out << indent;
196 writePoint(*it);
197 }
198 }
199 else if (field_ == "index" ) {
200 for (size_t n=begin; n<end; ++n) {
201 if (n!=0) {
202 out << join;
203 }
204 out << indent;
205 out << n + field_base_;
206 }
207 }
208 else if (field_ == "partition" ) {
209 for (size_t n=begin; n<end; ++n) {
210 if (n!=0) {
211 out << join;
212 }
213 out << indent;
214 out << distribution_.partition(n) + field_base_;
215 }
216 }
217 else {
218 ATLAS_THROW_EXCEPTION("Cannot output field \""<<field_<<"\"");
219 }
220 }
221 write_progress_end();
222 }
223
224 if (pretty_) {
225 out << '\n';
226 }
227 out << "]";
228 out << std::endl;
229 }
230 else {
231 std::vector<int> part(std::min<size_t>(chunk_size,grid_.size()));
232 auto write_chunk = [&](int partition, size_t begin, size_t end, size_t& n) {
233 size_t size{end - begin};
234 if (partition != -1) {
235 distribution_.partition(begin, end, part);
236 auto part_begin = part.begin();
237 auto part_end = part.begin()+size;
238 if (std::find(part_begin,part_end,partition) == part_end) {
239 return; // no points in this chunk to write
240 }
241 }
242 else {
243 part.assign(part.size(),-1);
244 }
245 std::string join = points_newline ? ",\n" : ",";
246 std::string indent(points_indent,' ');
247 if( field_ == "lonlat" ) {
248 auto it = grid_.lonlat().begin() + begin;
249 for (size_t j=0; j<size; ++j, ++it) {
250 if (part[j] == partition) {
251 if (n!=0) {
252 out << join;
253 }
254 out << indent;
255 writePoint(*it);
256 ++n;
257 }
258 }
259 }
260 else if (field_ == "index") {
261 for (size_t j=0; j<size; ++j) {
262 if (part[j] == partition) {
263 if (n!=0) {
264 out << join;
265 }
266 out << indent;
267 out << begin + j + field_base_;
268 ++n;
269 }
270 }
271 }
272 else {
273 ATLAS_THROW_EXCEPTION("Cannot output field \""<<field_<<"\"");
274 }
275 };
276
277 auto write_partition = [&](int partition) {
278 if (info && verbose_) {
279 *info << "Partition " << partition << std::endl;
280 }
281 size_t end = 0;
282 size_t n{0};
283 out << std::string(partition_indent,' ') << "[";
284 if (pretty_) {
285 out << '\n';
286 }
287 while(end != grid_.size()) {
288 auto begin = end;
289 if (info && verbose_ && grid_.size() > chunk_size) {
290 *info << std::fixed << std::setprecision(1) << double(begin)/double(grid_.size()) * 100 << "% completed" << std::endl;
291 }
292 end = std::min<size_t>(grid_.size(),begin+chunk_size);
293 write_chunk(partition,begin,end,n);
294 }
295 if (verbose_) {
296 if (info && grid_.size() > chunk_size) {
297 *info << "100% completed" << std::endl;
298 }
299 }
300 if (pretty_) {
301 out << '\n';
302 }
303 out << std::string(partition_indent,' ') << "]" << std::flush;
304 return n;
305 };
306
307 if (partition_ >= 0) {
308 auto n = write_partition(partition_);
309 out << std::endl;
310 if (info) {
311 *info << "Partition " << partition_ << " contains " << n << " points." << std::endl;
312 }
313 }
314 else {
315 if (pretty_) {
316 partition_indent += 2;
317 points_indent += 2;
318 }
319 out << "[\n";
320 std::vector<size_t> points_per_partition(nb_partitions_);
321 for( int p = 0; p < nb_partitions_; ++p ) {
322 if (p != 0) {
323 out << ",\n";
324 }
325 points_per_partition[p] = write_partition(p);
326 }
327 out << "\n]";
328 out << std::endl;
329 if (info) {
330 *info << "Points per partition:" << std::endl;
331 for (size_t p=0; p<nb_partitions_; ++p) {
332 *info << " " << std::setw(5) << std::left << p << points_per_partition[p] << std::endl;
333 }
334 }
335 }
336 }
337 out.flags(saved_fmtflags);
338 }
339
340 //------------------------------------------------------------------------------
341
342 }
343 }