| 1 |
#!/usr/bin/env python3 |
| 2 |
|
| 3 |
# Copyright (C) 2018-2020 Simon Spöhel |
| 4 |
# |
| 5 |
# This program is free software: you can redistribute it and/or modify |
| 6 |
# it under the terms of the GNU General Public License as published by |
| 7 |
# the Free Software Foundation, either version 3 of the License, or |
| 8 |
# (at your option) any later version. |
| 9 |
# |
| 10 |
# This program is distributed in the hope that it will be useful, |
| 11 |
# but WITHOUT ANY WARRANTY; without even the implied warranty of |
| 12 |
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the |
| 13 |
# GNU General Public License for more details. |
| 14 |
# |
| 15 |
# You should have received a copy of the GNU General Public License |
| 16 |
# along with this program. If not, see <http://www.gnu.org/licenses/>. |
| 17 |
|
| 18 |
import logging |
| 19 |
import argparse |
| 20 |
import os |
| 21 |
import datetime as dt |
| 22 |
import glob |
| 23 |
import sys |
| 24 |
import hashlib |
| 25 |
import configparser |
| 26 |
|
| 27 |
from collections import OrderedDict |
| 28 |
from sentinelsat import SentinelAPI, read_geojson, geojson_to_wkt |
| 29 |
|
| 30 |
import sentinel |
| 31 |
|
| 32 |
def file_exists(fname, stop=None): |
| 33 |
if not os.path.exists(fname): |
| 34 |
logging.warning('Cannot find file "%s".' % fname) |
| 35 |
if stop is not None: |
| 36 |
print('Exiting.') |
| 37 |
sys.exit(1) |
| 38 |
return 1 |
| 39 |
return 0 |
| 40 |
|
| 41 |
|
| 42 |
|
| 43 |
def print_config( config ): |
| 44 |
return_list = list() |
| 45 |
for section in config.sections(): |
| 46 |
return_list.append('[%s]' %section) |
| 47 |
for key, value in config.items(section): |
| 48 |
return_list.append('\t%s = %s' %(key, value)) |
| 49 |
return '\n'.join(return_list) |
| 50 |
|
| 51 |
|
| 52 |
|
| 53 |
def read_configs(conf_files, keys): |
| 54 |
'''Reads config files from a list of files. |
| 55 |
Adds path information for all options in "keys"-list. |
| 56 |
Returns a ConfigParser object. |
| 57 |
''' |
| 58 |
config = configparser.ConfigParser() |
| 59 |
for conf in conf_files: |
| 60 |
cfg = configparser.ConfigParser() |
| 61 |
cfg.read(conf) |
| 62 |
for section in cfg.sections(): |
| 63 |
for key in keys: |
| 64 |
if cfg.has_option(section, key): |
| 65 |
cfg[section][key] = '%s/%s' %(os.path.dirname(conf), cfg[section][key]) |
| 66 |
config.read_dict( cfg ) |
| 67 |
return config |
| 68 |
|
| 69 |
|
| 70 |
|
| 71 |
def parse_global( config ): |
| 72 |
'''Parses the global part of a config file, creates a dictionary and |
| 73 |
returns it.''' |
| 74 |
|
| 75 |
if not config.has_section('global'): |
| 76 |
logging.critical('Found no section "global" in config file, exiting.') |
| 77 |
sys.exit(1) |
| 78 |
|
| 79 |
if not config.has_section('credentials'): |
| 80 |
logging.critical('Found no section "credentials" in config file, exiting.') |
| 81 |
sys.exit(1) |
| 82 |
|
| 83 |
settings = dict() |
| 84 |
|
| 85 |
try: |
| 86 |
settings['base_dest'] = config['global']['base_dest'] |
| 87 |
except: |
| 88 |
logging.critical('could not parse "base_dest".') |
| 89 |
sys.exit(1) |
| 90 |
|
| 91 |
try: |
| 92 |
settings['downloaded_files'] = config['global']['downloaded_files'] |
| 93 |
except: |
| 94 |
logging.critical('could not parse "downloaded_files".') |
| 95 |
sys.exit(1) |
| 96 |
|
| 97 |
try: |
| 98 |
settings['platformname'] = config['global']['platformname'] |
| 99 |
except: |
| 100 |
logging.critical('could not parse "platformname".') |
| 101 |
sys.exit(1) |
| 102 |
|
| 103 |
try: |
| 104 |
settings['username'] = config['credentials']['username'] |
| 105 |
except: |
| 106 |
logging.critical('could not parse "username".') |
| 107 |
sys.exit(1) |
| 108 |
|
| 109 |
try: |
| 110 |
settings['password'] = config['credentials']['password'] |
| 111 |
except: |
| 112 |
logging.critical('could not parse "password".') |
| 113 |
sys.exit(1) |
| 114 |
|
| 115 |
return settings |
| 116 |
|
| 117 |
|
| 118 |
|
| 119 |
def parse_dlsettings(config): |
| 120 |
'''Parses a config file without the global part. Creates a dictionary |
| 121 |
with per region with a nested dictrionary for each region and returns |
| 122 |
it.''' |
| 123 |
|
| 124 |
if len(config.sections()) < 1: |
| 125 |
logging.critical('Found no sections in config file, exiting.') |
| 126 |
sys.exit(1) |
| 127 |
|
| 128 |
dlsettings = dict() |
| 129 |
|
| 130 |
logging.info('Parsing dlsettings.') |
| 131 |
|
| 132 |
for section in config.sections(): |
| 133 |
|
| 134 |
try: |
| 135 |
date_start = dt.datetime.strptime(config[section]['date_start'], '%Y-%m-%d').date() |
| 136 |
except: |
| 137 |
logging.warning('could not parse "date_start", got %s' %config[section]['date_start']) |
| 138 |
|
| 139 |
try: |
| 140 |
date_end = dt.datetime.strptime(config[section]['date_end'], '%Y-%m-%d').date() |
| 141 |
except ValueError: |
| 142 |
if config[section]['date_end'] == 'today': |
| 143 |
date_end = dt.date.today() |
| 144 |
else: |
| 145 |
logging.warning('Could not parse "date_end", got "%s".' %config[section]['date_end']) |
| 146 |
return |
| 147 |
|
| 148 |
try: |
| 149 |
clouds_min = config[section].getint('clouds_min') |
| 150 |
except ValueError: |
| 151 |
logging.warning('Could not parse "clouds_min", got "%s".' %config[section]['clouds_min']) |
| 152 |
return |
| 153 |
|
| 154 |
try: |
| 155 |
clouds_max = config[section].getint('clouds_max') |
| 156 |
except ValueError: |
| 157 |
logging.warning('Could not parse "clouds_max", got "%s".' %config[section]['clouds_max']) |
| 158 |
return |
| 159 |
|
| 160 |
try: |
| 161 |
footprint = config[section]['footprint'] |
| 162 |
except: |
| 163 |
logging.warning('Could not parse "footprint", got "%s".' %config[section]['footprint']) |
| 164 |
return |
| 165 |
|
| 166 |
try: |
| 167 |
product_type = config[section]['product_type'] |
| 168 |
except: |
| 169 |
logging.warning('Could not parse "product_type", got "%s".' %config[section]['product_type']) |
| 170 |
return |
| 171 |
|
| 172 |
cfg = dict() |
| 173 |
cfg['date_start'] = date_start |
| 174 |
cfg['date_end' ] = date_end |
| 175 |
cfg['clouds_min'] = clouds_min |
| 176 |
cfg['clouds_max'] = clouds_max |
| 177 |
cfg['footprint' ] = footprint |
| 178 |
cfg['product_type' ] = product_type |
| 179 |
|
| 180 |
dlsettings[section] = cfg |
| 181 |
return dlsettings |
| 182 |
|
| 183 |
|
| 184 |
|
| 185 |
def load_ignores(ignore_file): |
| 186 |
'''Loads the ignores file and returns a list.''' |
| 187 |
|
| 188 |
if file_exists(ignore_file) is 0: |
| 189 |
logging.info('Loading ignore list.') |
| 190 |
|
| 191 |
with open(ignore_file) as f: |
| 192 |
ignores_string = f.read() |
| 193 |
|
| 194 |
ignores = ignores_string.split('\n') |
| 195 |
ignores = [x for x in ignores if len(x) > 0] |
| 196 |
ignores = [x for x in ignores if x[0] != '#'] |
| 197 |
return ignores |
| 198 |
|
| 199 |
|
| 200 |
|
| 201 |
def filter_products(df, ignorelist): |
| 202 |
'''Filters all items of an ignore list out of a dataframe |
| 203 |
and returns a list.''' |
| 204 |
|
| 205 |
df_filtered = df[ -df['filename'].isin(ignorelist)] |
| 206 |
return df_filtered.index.tolist() |
| 207 |
|
| 208 |
|
| 209 |
|
| 210 |
parser = argparse.ArgumentParser(description='Download sentinel 2 data') |
| 211 |
|
| 212 |
parser.add_argument('-a', '--auto', |
| 213 |
help='use settings config files, implies -d', |
| 214 |
action='store_true') |
| 215 |
|
| 216 |
parser.add_argument('--debug', |
| 217 |
choices=['debug','info','warning','critical'], |
| 218 |
help='set the debug level.') |
| 219 |
|
| 220 |
parser.add_argument('-d', '--download', |
| 221 |
help='set this download the data', |
| 222 |
action='store_true') |
| 223 |
|
| 224 |
if len(sys.argv)==1: |
| 225 |
parser.print_help(sys.stderr) |
| 226 |
sys.exit(1) |
| 227 |
|
| 228 |
args = parser.parse_args() |
| 229 |
|
| 230 |
if args.debug is not None: |
| 231 |
loglevel = args.debug |
| 232 |
args.debug = None |
| 233 |
else: |
| 234 |
loglevel = 'WARNING' |
| 235 |
|
| 236 |
numeric_level = getattr(logging, loglevel.upper(), None) |
| 237 |
if not isinstance(numeric_level, int): |
| 238 |
raise VelueError('Invalid log level: %s' % loglevel) |
| 239 |
logging.basicConfig(stream=sys.stdout, level=numeric_level) |
| 240 |
|
| 241 |
args_supplied = [x for x in vars(args).values() if x not in [None, False] ] |
| 242 |
|
| 243 |
|
| 244 |
logging.info('New run begins at %s' %dt.datetime.now()) |
| 245 |
|
| 246 |
dumpconf_list = ['All configuration known to this run', |
| 247 |
'==================================='] |
| 248 |
# only read conf files if arg.auto == True |
| 249 |
if args.auto is True: |
| 250 |
logging.debug('auto was supplied, setting download = True.') |
| 251 |
args.download = True |
| 252 |
logging.debug('auto was supplied, parsing conf files.') |
| 253 |
# conf files |
| 254 |
settings_locations = [ '/etc/sentineldl', 'settings' ] |
| 255 |
|
| 256 |
# FIXME catch errors here. |
| 257 |
for item in settings_locations: |
| 258 |
if file_exists( item) == 0: |
| 259 |
settings_location = item |
| 260 |
break |
| 261 |
|
| 262 |
logging.info('Using settings location %s' %settings_location) |
| 263 |
conf_files = glob.glob('%s/**/*.conf' %settings_location, recursive=True) |
| 264 |
logging.debug('Parsing config files: %s.' %', '.join(conf_files)) |
| 265 |
config = read_configs(conf_files, ['footprint','credentials_file']) |
| 266 |
dumpconf_list.append(print_config(config)) |
| 267 |
|
| 268 |
cfg_settings = parse_global(config) |
| 269 |
else: |
| 270 |
logging.critical('--auto is required at the moment') |
| 271 |
sys.exit(1) |
| 272 |
|
| 273 |
settings = cfg_settings |
| 274 |
|
| 275 |
#print('settings: ', settings) |
| 276 |
dumpconf_list.append('\nsettings:') |
| 277 |
dumpconf_list.append('%s' %settings) |
| 278 |
|
| 279 |
|
| 280 |
# create dlsettings |
| 281 |
dlsettings = config |
| 282 |
del dlsettings['global'] |
| 283 |
del dlsettings['credentials'] |
| 284 |
|
| 285 |
dlconfig = parse_dlsettings(dlsettings) |
| 286 |
|
| 287 |
dumpconf_list.append('\ndlconfig:') |
| 288 |
dumpconf_list.append('%s' %dlconfig) |
| 289 |
|
| 290 |
dumpconf_list.append('===================================') |
| 291 |
logging.debug('\n'.join(dumpconf_list)) |
| 292 |
|
| 293 |
regions = list(dlconfig.keys()) |
| 294 |
|
| 295 |
# read ignores |
| 296 |
ignore_files = glob.glob('%s/**/ignore.txt' %settings_location, recursive=True) |
| 297 |
|
| 298 |
ignores = list() |
| 299 |
for f in ignore_files: |
| 300 |
ignores.extend(load_ignores(f)) |
| 301 |
|
| 302 |
# remove duplicated entries in ignores |
| 303 |
ignores = list(set(ignores)) |
| 304 |
|
| 305 |
sapi = SentinelAPI(settings['username'], settings['password'], show_progressbars=False) |
| 306 |
|
| 307 |
for region_name in regions: |
| 308 |
cfg = dlconfig[region_name] |
| 309 |
|
| 310 |
footprint = geojson_to_wkt(read_geojson( cfg['footprint'] )) |
| 311 |
|
| 312 |
logging.info('Region: %s, product: %s' %(region_name, cfg['product_type'])) |
| 313 |
|
| 314 |
metadata = sentinel.discover( |
| 315 |
username = settings['username'], |
| 316 |
password = settings['password'], |
| 317 |
footprint = cfg['footprint'], |
| 318 |
platformname = settings['platformname'], |
| 319 |
producttype = cfg['product_type'], |
| 320 |
dates = (cfg['date_start'], cfg['date_end']), |
| 321 |
clouds = (cfg['clouds_min'], cfg['clouds_max']), |
| 322 |
debug_save = False, |
| 323 |
) |
| 324 |
|
| 325 |
if len(metadata) < 1: |
| 326 |
logging.info('Nothing to download.') |
| 327 |
continue |
| 328 |
|
| 329 |
metadata = metadata[~metadata.index.isin(ignores)] |
| 330 |
|
| 331 |
logging.info('%i products remaining for download.' %len(metadata.index) ) |
| 332 |
|
| 333 |
if args.download != True: |
| 334 |
logging.warning('Skipping download. Use --download to avoid this message.') |
| 335 |
continue |
| 336 |
|
| 337 |
for index, row in metadata.iterrows(): |
| 338 |
sentinel.download( |
| 339 |
username = settings['username'], |
| 340 |
password = settings['password'], |
| 341 |
base_dest= settings['base_dest'], |
| 342 |
downloaded_files = settings['downloaded_files'], |
| 343 |
index = index, |
| 344 |
row = row, |
| 345 |
debug_save=False) |
| 346 |
|
| 347 |
# kate: indent-mode python; indent-pasted-text false; indent-width 4; replace-tabs true; tab-width 4; |