Debian Codegraph @

lib/sentineldl

raw
1 #!/usr/bin/env python3
2
3 # Copyright (C) 2018-2020 Simon Spöhel
4 #
5 # This program is free software: you can redistribute it and/or modify
6 # it under the terms of the GNU General Public License as published by
7 # the Free Software Foundation, either version 3 of the License, or
8 # (at your option) any later version.
9 #
10 # This program is distributed in the hope that it will be useful,
11 # but WITHOUT ANY WARRANTY; without even the implied warranty of
12 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13 # GNU General Public License for more details.
14 #
15 # You should have received a copy of the GNU General Public License
16 # along with this program. If not, see <http://www.gnu.org/licenses/>.
17
18 import logging
19 import argparse
20 import os
21 import datetime as dt
22 import glob
23 import sys
24 import hashlib
25 import configparser
26
27 from collections import OrderedDict
28 from sentinelsat import SentinelAPI, read_geojson, geojson_to_wkt
29
30 import sentinel
31
32 def file_exists(fname, stop=None):
33 if not os.path.exists(fname):
34 logging.warning('Cannot find file "%s".' % fname)
35 if stop is not None:
36 print('Exiting.')
37 sys.exit(1)
38 return 1
39 return 0
40
41
42
43 def print_config( config ):
44 return_list = list()
45 for section in config.sections():
46 return_list.append('[%s]' %section)
47 for key, value in config.items(section):
48 return_list.append('\t%s = %s' %(key, value))
49 return '\n'.join(return_list)
50
51
52
53 def read_configs(conf_files, keys):
54 '''Reads config files from a list of files.
55 Adds path information for all options in "keys"-list.
56 Returns a ConfigParser object.
57 '''
58 config = configparser.ConfigParser()
59 for conf in conf_files:
60 cfg = configparser.ConfigParser()
61 cfg.read(conf)
62 for section in cfg.sections():
63 for key in keys:
64 if cfg.has_option(section, key):
65 cfg[section][key] = '%s/%s' %(os.path.dirname(conf), cfg[section][key])
66 config.read_dict( cfg )
67 return config
68
69
70
71 def parse_global( config ):
72 '''Parses the global part of a config file, creates a dictionary and
73 returns it.'''
74
75 if not config.has_section('global'):
76 logging.critical('Found no section "global" in config file, exiting.')
77 sys.exit(1)
78
79 if not config.has_section('credentials'):
80 logging.critical('Found no section "credentials" in config file, exiting.')
81 sys.exit(1)
82
83 settings = dict()
84
85 try:
86 settings['base_dest'] = config['global']['base_dest']
87 except:
88 logging.critical('could not parse "base_dest".')
89 sys.exit(1)
90
91 try:
92 settings['downloaded_files'] = config['global']['downloaded_files']
93 except:
94 logging.critical('could not parse "downloaded_files".')
95 sys.exit(1)
96
97 try:
98 settings['platformname'] = config['global']['platformname']
99 except:
100 logging.critical('could not parse "platformname".')
101 sys.exit(1)
102
103 try:
104 settings['username'] = config['credentials']['username']
105 except:
106 logging.critical('could not parse "username".')
107 sys.exit(1)
108
109 try:
110 settings['password'] = config['credentials']['password']
111 except:
112 logging.critical('could not parse "password".')
113 sys.exit(1)
114
115 return settings
116
117
118
119 def parse_dlsettings(config):
120 '''Parses a config file without the global part. Creates a dictionary
121 with per region with a nested dictrionary for each region and returns
122 it.'''
123
124 if len(config.sections()) < 1:
125 logging.critical('Found no sections in config file, exiting.')
126 sys.exit(1)
127
128 dlsettings = dict()
129
130 logging.info('Parsing dlsettings.')
131
132 for section in config.sections():
133
134 try:
135 date_start = dt.datetime.strptime(config[section]['date_start'], '%Y-%m-%d').date()
136 except:
137 logging.warning('could not parse "date_start", got %s' %config[section]['date_start'])
138
139 try:
140 date_end = dt.datetime.strptime(config[section]['date_end'], '%Y-%m-%d').date()
141 except ValueError:
142 if config[section]['date_end'] == 'today':
143 date_end = dt.date.today()
144 else:
145 logging.warning('Could not parse "date_end", got "%s".' %config[section]['date_end'])
146 return
147
148 try:
149 clouds_min = config[section].getint('clouds_min')
150 except ValueError:
151 logging.warning('Could not parse "clouds_min", got "%s".' %config[section]['clouds_min'])
152 return
153
154 try:
155 clouds_max = config[section].getint('clouds_max')
156 except ValueError:
157 logging.warning('Could not parse "clouds_max", got "%s".' %config[section]['clouds_max'])
158 return
159
160 try:
161 footprint = config[section]['footprint']
162 except:
163 logging.warning('Could not parse "footprint", got "%s".' %config[section]['footprint'])
164 return
165
166 try:
167 product_type = config[section]['product_type']
168 except:
169 logging.warning('Could not parse "product_type", got "%s".' %config[section]['product_type'])
170 return
171
172 cfg = dict()
173 cfg['date_start'] = date_start
174 cfg['date_end' ] = date_end
175 cfg['clouds_min'] = clouds_min
176 cfg['clouds_max'] = clouds_max
177 cfg['footprint' ] = footprint
178 cfg['product_type' ] = product_type
179
180 dlsettings[section] = cfg
181 return dlsettings
182
183
184
185 def load_ignores(ignore_file):
186 '''Loads the ignores file and returns a list.'''
187
188 if file_exists(ignore_file) is 0:
189 logging.info('Loading ignore list.')
190
191 with open(ignore_file) as f:
192 ignores_string = f.read()
193
194 ignores = ignores_string.split('\n')
195 ignores = [x for x in ignores if len(x) > 0]
196 ignores = [x for x in ignores if x[0] != '#']
197 return ignores
198
199
200
201 def filter_products(df, ignorelist):
202 '''Filters all items of an ignore list out of a dataframe
203 and returns a list.'''
204
205 df_filtered = df[ -df['filename'].isin(ignorelist)]
206 return df_filtered.index.tolist()
207
208
209
210 parser = argparse.ArgumentParser(description='Download sentinel 2 data')
211
212 parser.add_argument('-a', '--auto',
213 help='use settings config files, implies -d',
214 action='store_true')
215
216 parser.add_argument('--debug',
217 choices=['debug','info','warning','critical'],
218 help='set the debug level.')
219
220 parser.add_argument('-d', '--download',
221 help='set this download the data',
222 action='store_true')
223
224 if len(sys.argv)==1:
225 parser.print_help(sys.stderr)
226 sys.exit(1)
227
228 args = parser.parse_args()
229
230 if args.debug is not None:
231 loglevel = args.debug
232 args.debug = None
233 else:
234 loglevel = 'WARNING'
235
236 numeric_level = getattr(logging, loglevel.upper(), None)
237 if not isinstance(numeric_level, int):
238 raise VelueError('Invalid log level: %s' % loglevel)
239 logging.basicConfig(stream=sys.stdout, level=numeric_level)
240
241 args_supplied = [x for x in vars(args).values() if x not in [None, False] ]
242
243
244 logging.info('New run begins at %s' %dt.datetime.now())
245
246 dumpconf_list = ['All configuration known to this run',
247 '===================================']
248 # only read conf files if arg.auto == True
249 if args.auto is True:
250 logging.debug('auto was supplied, setting download = True.')
251 args.download = True
252 logging.debug('auto was supplied, parsing conf files.')
253 # conf files
254 settings_locations = [ '/etc/sentineldl', 'settings' ]
255
256 # FIXME catch errors here.
257 for item in settings_locations:
258 if file_exists( item) == 0:
259 settings_location = item
260 break
261
262 logging.info('Using settings location %s' %settings_location)
263 conf_files = glob.glob('%s/**/*.conf' %settings_location, recursive=True)
264 logging.debug('Parsing config files: %s.' %', '.join(conf_files))
265 config = read_configs(conf_files, ['footprint','credentials_file'])
266 dumpconf_list.append(print_config(config))
267
268 cfg_settings = parse_global(config)
269 else:
270 logging.critical('--auto is required at the moment')
271 sys.exit(1)
272
273 settings = cfg_settings
274
275 #print('settings: ', settings)
276 dumpconf_list.append('\nsettings:')
277 dumpconf_list.append('%s' %settings)
278
279
280 # create dlsettings
281 dlsettings = config
282 del dlsettings['global']
283 del dlsettings['credentials']
284
285 dlconfig = parse_dlsettings(dlsettings)
286
287 dumpconf_list.append('\ndlconfig:')
288 dumpconf_list.append('%s' %dlconfig)
289
290 dumpconf_list.append('===================================')
291 logging.debug('\n'.join(dumpconf_list))
292
293 regions = list(dlconfig.keys())
294
295 # read ignores
296 ignore_files = glob.glob('%s/**/ignore.txt' %settings_location, recursive=True)
297
298 ignores = list()
299 for f in ignore_files:
300 ignores.extend(load_ignores(f))
301
302 # remove duplicated entries in ignores
303 ignores = list(set(ignores))
304
305 sapi = SentinelAPI(settings['username'], settings['password'], show_progressbars=False)
306
307 for region_name in regions:
308 cfg = dlconfig[region_name]
309
310 footprint = geojson_to_wkt(read_geojson( cfg['footprint'] ))
311
312 logging.info('Region: %s, product: %s' %(region_name, cfg['product_type']))
313
314 metadata = sentinel.discover(
315 username = settings['username'],
316 password = settings['password'],
317 footprint = cfg['footprint'],
318 platformname = settings['platformname'],
319 producttype = cfg['product_type'],
320 dates = (cfg['date_start'], cfg['date_end']),
321 clouds = (cfg['clouds_min'], cfg['clouds_max']),
322 debug_save = False,
323 )
324
325 if len(metadata) < 1:
326 logging.info('Nothing to download.')
327 continue
328
329 metadata = metadata[~metadata.index.isin(ignores)]
330
331 logging.info('%i products remaining for download.' %len(metadata.index) )
332
333 if args.download != True:
334 logging.warning('Skipping download. Use --download to avoid this message.')
335 continue
336
337 for index, row in metadata.iterrows():
338 sentinel.download(
339 username = settings['username'],
340 password = settings['password'],
341 base_dest= settings['base_dest'],
342 downloaded_files = settings['downloaded_files'],
343 index = index,
344 row = row,
345 debug_save=False)
346
347 # kate: indent-mode python; indent-pasted-text false; indent-width 4; replace-tabs true; tab-width 4;