Debian Codegraph @

frmts/grib/degrib/merge_degrib_and_wmo_tables.py

raw
1 #!/usr/bin/env python
2 # SPDX-License-Identifier: MIT
3 # Copyright (C) 2021 Even Rouault
4
5 """This scripts merges the GRIB2 tables from DEGRIB, which are in the
6 frmts/grib/degrib/data directory with the ones from WMO.
7 """
8
9 import csv
10 import os
11 import re
12 import shutil
13
14 import requests
15
16 # URL of the WMO GRIB2 tables repository
17 wmo_github_repo = "wmo-im/GRIB2"
18 wmo_github_tag = "v36"
19 wmo_base_url = f"https://raw.githubusercontent.com/{wmo_github_repo}/{wmo_github_tag}"
20
21 script_dir_name = os.path.dirname(os.path.realpath(__file__))
22
23 # Input directory with degrib tables
24 degrib_table_input_dir = os.path.join(script_dir_name, "data")
25 assert os.path.exists(degrib_table_input_dir)
26
27 # Output directory of merged tables (frmts/grib/data)
28 output_dir = os.path.join(os.path.dirname(script_dir_name), "data")
29 assert os.path.exists(output_dir)
30
31 with open(os.path.join(output_dir, "grib2_table_versions.csv"), "wt") as f:
32 f.write("component,version\n")
33 f.write(f"wmo,{wmo_github_tag}\n")
34 f.write("degrib,2.25\n")
35
36 r = requests.get(
37 f"https://api.github.com/repos/{wmo_github_repo}/commits/HEAD",
38 headers={"Accept": "application/vnd.github.VERSION.sha"},
39 )
40 wmo_table_sha1 = r.content.decode("utf-8")
41 warning_msg = "DO NOT MODIFY THIS FILE. It is generated by frmts/grib/degrib/merge_degrib_and_wmo_tables.py"
42 warning2_msg = (
43 f"from tables at version https://github.com/wmo-im/GRIB2/commit/{wmo_table_sha1}"
44 )
45 warning_len = max(len(warning_msg), len(warning2_msg))
46
47 # Processes table 4.5 "Fixed surface types and units"
48
49
50 def process_table_4_5():
51
52 # Ingest table from degrib
53 degrib_csv_reader = csv.DictReader(
54 open(os.path.join(degrib_table_input_dir, "grib2_table_4_5.csv"), newline="")
55 )
56 assert degrib_csv_reader.fieldnames == ["code", "short_name", "name", "unit"]
57 records = [None for i in range(256)]
58 for row in degrib_csv_reader:
59 code = int(row["code"])
60 assert records[code] is None
61 row["code"] = code
62 records[code] = row
63
64 # Ingest table from WMO
65 r = requests.get(wmo_base_url + "/GRIB2_CodeFlag_4_5_CodeTable_en.csv")
66 lines = r.content.decode("utf-8").split("\n")
67 wmo_csv_reader = csv.DictReader(lines)
68 assert wmo_csv_reader.fieldnames == [
69 "Title_en",
70 "SubTitle_en",
71 "CodeFlag",
72 "Value",
73 "MeaningParameterDescription_en",
74 "Note_en",
75 "noteIDs",
76 "UnitComments_en",
77 "Status",
78 ], wmo_csv_reader.fieldnames
79 wmo_records = [None for i in range(256)]
80 for row in wmo_csv_reader:
81 code = row["CodeFlag"]
82 if "-" in code:
83 # Deal with ranges (reserved codes)
84 min_val, max_val = code.split("-")
85 min_val = int(min_val)
86 max_val = int(max_val)
87 assert min_val < max_val
88 else:
89 min_val = int(code)
90 max_val = int(code)
91 for code in range(min_val, max_val + 1):
92 assert wmo_records[code] is None
93 wmo_records[code] = {
94 "code": code,
95 "name": row["MeaningParameterDescription_en"],
96 "unit": row["UnitComments_en"],
97 }
98
99 # Merge records from WMO into degrib
100 # degrib ones have the precedence when they exist since they contain
101 # short_name.
102 for code in range(len(records)):
103 if wmo_records[code] is not None:
104 if records[code] is None or (
105 "Reserved" in records[code]["name"]
106 and "Reserved" not in wmo_records[code]["name"]
107 ):
108 unit = wmo_records[code]["unit"]
109 unit = "-" if unit == "" else unit
110 records[code] = {
111 "code": code,
112 "short_name": "",
113 "name": wmo_records[code]["name"],
114 "unit": unit,
115 }
116
117 # Write the final file
118 with open(
119 os.path.join(output_dir, "grib2_table_4_5.csv"), "w", newline=""
120 ) as csvfile:
121 writer = csv.DictWriter(
122 csvfile,
123 fieldnames=degrib_csv_reader.fieldnames,
124 quoting=csv.QUOTE_NONNUMERIC,
125 )
126 writer.writeheader()
127 writer.writerow(
128 {"code": -4, "short_name": "#" * warning_len, "name": "#", "unit": "#"}
129 )
130 writer.writerow(
131 {"code": -3, "short_name": warning_msg, "name": "#", "unit": "#"}
132 )
133 writer.writerow(
134 {"code": -2, "short_name": warning2_msg, "name": "#", "unit": "#"}
135 )
136 writer.writerow(
137 {"code": -1, "short_name": "#" * warning_len, "name": "#", "unit": "#"}
138 )
139 for row in records:
140 writer.writerow(row)
141
142
143 process_table_4_5()
144
145
146 # Process table referenced by 4.2 with "(Code table 4.X)" in the UnitComments_en
147 # column
148 def get_auxiliary_table(num):
149 url = wmo_base_url + f"/GRIB2_CodeFlag_4_{num}_CodeTable_en.csv"
150 print(f"Fetching {url}...")
151 r = requests.get(url)
152 lines = r.content.decode("utf-8").split("\n")
153 wmo_csv_reader = csv.DictReader(lines)
154 assert wmo_csv_reader.fieldnames == [
155 "Title_en",
156 "SubTitle_en",
157 "CodeFlag",
158 "Value",
159 "MeaningParameterDescription_en",
160 "Note_en",
161 "noteIDs",
162 "UnitComments_en",
163 "Status",
164 ], wmo_csv_reader.fieldnames
165 ret = ""
166 for row in wmo_csv_reader:
167 if ret:
168 ret += "; "
169 ret += row["CodeFlag"]
170 ret += "="
171 ret += row["MeaningParameterDescription_en"]
172 return ret
173
174
175 # Processes table 4.2 "Parameter Number by Product discipline and Parameter category"
176
177
178 def process_table_4_2():
179
180 if os.path.exists("tmp_GRIB2"):
181 shutil.rmtree("tmp_GRIB2")
182 os.system(
183 f"git clone --branch {wmo_github_tag} https://github.com/{wmo_github_repo} tmp_GRIB2"
184 )
185 lines = []
186 # Merge together GRIB2_CodeFlag_4_2_xxxx.csv files
187 for filename in os.listdir("tmp_GRIB2"):
188 if filename.startswith("GRIB2_CodeFlag_4_2_"):
189 new_lines = (
190 open("tmp_GRIB2/" + filename, "rb").read().decode("utf-8").split("\n")
191 )
192 if len(lines) == 0:
193 lines = new_lines
194 else:
195 lines += new_lines[1:]
196 shutil.rmtree("tmp_GRIB2")
197
198 wmo_csv_reader = csv.DictReader(lines)
199 assert wmo_csv_reader.fieldnames == [
200 "Title_en",
201 "SubTitle_en",
202 "CodeFlag",
203 "Value",
204 "MeaningParameterDescription_en",
205 "Note_en",
206 "noteIDs",
207 "UnitComments_en",
208 "Status",
209 ], wmo_csv_reader.fieldnames
210
211 # dict_tables has as key a subtable (by product discipline and category)
212 # and as value a table of 256 entries with the subcategory as index
213 dict_tables = {}
214
215 auxiliary_tables = {}
216
217 for row in wmo_csv_reader:
218
219 # Split values
220 # "Product discipline 0 - Meteorological products, parameter category 1: moisture""
221 # as product_type = 0 and category = 1
222 m = re.match(r".+?([0-9]+).+?([0-9]+).+", row["SubTitle_en"])
223 product_type = int(m.group(1))
224 category = int(m.group(2))
225
226 # Instantiate final table
227 table_name = f"grib2_table_4_2_{product_type}_{category}.csv"
228 if table_name not in dict_tables:
229 dict_tables[table_name] = [None for i in range(256)]
230 table = dict_tables[table_name]
231
232 code = row["CodeFlag"]
233
234 if "-" in code:
235 min_val, max_val = code.split("-")
236 min_val = int(min_val)
237 max_val = int(max_val)
238 assert min_val < max_val
239 else:
240 min_val = int(code)
241 max_val = int(code)
242
243 name = row["MeaningParameterDescription_en"]
244 unit = row["UnitComments_en"]
245 m = re.match(r"\(Code table 4\.([0-9]+)\)", unit)
246 if m is not None:
247 auxiliary_table_num = m.group(1)
248 if auxiliary_table_num not in auxiliary_tables:
249 auxiliary_tables[auxiliary_table_num] = get_auxiliary_table(
250 auxiliary_table_num
251 )
252 unit = auxiliary_tables[auxiliary_table_num]
253
254 for code in range(min_val, max_val + 1):
255 assert table[code] is None, (table_name, code, table[code], row)
256 table[code] = {
257 "subcat": code,
258 "short_name": "",
259 "name": name,
260 "unit": unit,
261 "unit_conv": "UC_NONE",
262 "override_unit": True if m is not None else False,
263 }
264
265 fieldnames = ["subcat", "short_name", "name", "unit", "unit_conv"]
266
267 # Iterate over all sub tables.
268 for filename, wmo_table in dict_tables.items():
269 degrib_filename = os.path.join(degrib_table_input_dir, filename)
270
271 # Ingest corresponding degrib subtable, if it exists
272 if os.path.exists(degrib_filename):
273 degrib_csv_reader = csv.DictReader(open(degrib_filename, newline=""))
274 assert degrib_csv_reader.fieldnames == fieldnames
275 records = [None for i in range(256)]
276 for row in degrib_csv_reader:
277 code = int(row["subcat"])
278 assert records[code] is None
279 row["subcat"] = code
280 records[code] = row
281
282 # Merge WMO and degrib subtable
283 # degrib entries have precedence, unless there are reserved ranges
284 # and WMO entry is not reserved
285 for code in range(len(records)):
286 if wmo_table[code] is not None:
287 if records[code] is None:
288 records[code] = wmo_table[code]
289 elif (
290 "Reserved" in records[code]["name"]
291 and "Reserved" not in wmo_table[code]["name"]
292 ):
293 # print('Patching ', records[code], ' with ', wmo_table[code])
294 records[code] = wmo_table[code]
295 if wmo_table[code]["override_unit"]:
296 records[code]["unit"] = wmo_table[code]["unit"]
297
298 else:
299 records = wmo_table
300
301 # Write final subtable
302 with open(os.path.join(output_dir, filename), "w", newline="") as csvfile:
303 writer = csv.DictWriter(
304 csvfile, fieldnames=fieldnames, quoting=csv.QUOTE_NONNUMERIC
305 )
306 writer.writeheader()
307 writer.writerow(
308 {
309 "subcat": -4,
310 "short_name": "#" * warning_len,
311 "name": "#",
312 "unit": "#",
313 "unit_conv": "#",
314 }
315 )
316 writer.writerow(
317 {
318 "subcat": -3,
319 "short_name": warning_msg,
320 "name": "#",
321 "unit": "#",
322 "unit_conv": "#",
323 }
324 )
325 writer.writerow(
326 {
327 "subcat": -2,
328 "short_name": warning2_msg,
329 "name": "#",
330 "unit": "#",
331 "unit_conv": "#",
332 }
333 )
334 writer.writerow(
335 {
336 "subcat": -1,
337 "short_name": "#" * warning_len,
338 "name": "#",
339 "unit": "#",
340 "unit_conv": "#",
341 }
342 )
343 for row in records:
344 if row:
345 writer.writerow({x: row[x] for x in fieldnames})
346
347
348 process_table_4_2()