Debian Codegraph @

asf_search/download/download.py

raw
1 from typing import Iterable
2 from multiprocessing import Pool
3 import os.path
4 from urllib import parse
5 from requests import Response
6 from requests.exceptions import HTTPError
7 import warnings
8
9 from asf_search.exceptions import ASFAuthenticationError, ASFDownloadError
10 from asf_search import ASFSession
11 from tenacity import retry, stop_after_delay, retry_if_result, wait_fixed
12
13 try:
14 from remotezip import RemoteZip
15 except ImportError:
16 RemoteZip = None
17
18
19 def _download_url(arg):
20 url, path, session = arg
21 download_url(url=url, path=path, session=session)
22
23
24 def download_urls(urls: Iterable[str], path: str, session: ASFSession = None, processes: int = 1):
25 """
26 Downloads all products from the specified URLs to the specified location.
27
28 :param urls: List of URLs from which to download
29 :param path: Local path in which to save the product
30 :param session: The session to use, in most cases should be authenticated beforehand
31 :param processes: Number of download processes to use. Defaults to 1 (i.e. sequential download)
32 :return:
33 """
34 if session is None:
35 session = ASFSession()
36
37 if processes <= 1:
38 for url in urls:
39 download_url(url=url, path=path, session=session)
40 else:
41 pool = Pool(processes=processes)
42 args = [(url, path, session) for url in urls]
43 pool.map(_download_url, args)
44 pool.close()
45 pool.join()
46
47
48 def download_url(url: str, path: str, filename: str = None, session: ASFSession = None) -> None:
49 """
50 Downloads a product from the specified URL to the specified location and (optional) filename.
51
52 :param url: URL from which to download
53 :param path: Local path in which to save the product
54 :param filename: Optional filename to be used, extracted from the URL by default
55 :param session: The session to use, in most cases should be authenticated beforehand
56 :return:
57 """
58
59 if filename is None:
60 filename = os.path.split(parse.urlparse(url).path)[1]
61
62 if not os.path.isdir(path):
63 raise ASFDownloadError(f'Error downloading {url}: directory not found: {path}')
64
65 if os.path.isfile(os.path.join(path, filename)):
66 warnings.warn(f'File already exists, skipping download: {os.path.join(path, filename)}')
67 return
68
69 if session is None:
70 session = ASFSession()
71
72 response = _try_get_response(session=session, url=url)
73
74 with open(os.path.join(path, filename), 'wb') as f:
75 for chunk in response.iter_content(chunk_size=8192):
76 f.write(chunk)
77
78
79 def remotezip(url: str, session: ASFSession) -> 'RemoteZip': # type: ignore # noqa: F821
80 """
81 :param url: the url to the zip product
82 :param session: the authenticated ASFSession to read and download from the zip file
83 """
84 if RemoteZip is None:
85 raise ImportError(
86 'Could not find remotezip package in current python environment. '
87 '"remotezip" is an optional dependency of asf-search required '
88 'for the `remotezip()` method. '
89 'Enable by including the appropriate pip or conda install. '
90 'Ex: `python3 -m pip install asf-search[extras]`'
91 )
92
93 session.hooks['response'].append(strip_auth_if_aws)
94 return RemoteZip(url, session=session)
95
96
97 def strip_auth_if_aws(r, *args, **kwargs):
98 if (
99 300 <= r.status_code <= 399
100 and 'amazonaws.com' in parse.urlparse(r.headers['location']).netloc
101 ):
102 location = r.headers['location']
103 r.headers.clear()
104 r.headers['location'] = location
105
106
107 # if it's an unprocessed burst product it'll return a 202 and we'll have to query again
108 # https://sentinel1-burst-docs.asf.alaska.edu/
109 def _is_burst_processing(response: Response):
110 return response.status_code == 202
111
112
113 @retry(
114 reraise=True,
115 retry=retry_if_result(_is_burst_processing),
116 wait=wait_fixed(1),
117 stop=stop_after_delay(90),
118 )
119 def _try_get_response(session: ASFSession, url: str):
120 response = session.get(url, stream=True, hooks={'response': strip_auth_if_aws})
121
122 try:
123 response.raise_for_status()
124 except HTTPError as e:
125 if 400 <= response.status_code <= 499:
126 raise ASFAuthenticationError(f'HTTP {e.response.status_code}: {e.response.text}')
127
128 raise e
129
130 return response