Debian Codegraph @

fd_tool/fd_tool/api/releases.py

raw ◀ ▶
1 Debian patch touch this file
1 """Collect all information about released dictionaries."""
2
3 from datetime import datetime
4 import hashlib
5 import json
6 import os
7 import re
8 import shutil
9 import subprocess
10 import sys
11 import urllib.request
12
13 import semver
14
15 from .config import RELEASE_HTTP_TOOL_BASE
16 from .dictionary import DownloadFormat, normalize_version
17
18 # relative to github.com/freedict/
19 TOOLS_REPO = 'tools'
20 USER_AGENT = 'Friendly FreeDict Helper'
21
22 class ReleaseError(Exception):
23 """This error can occur, when information about a release is gathered. It
24 wraps all sorts of ValueErrors and IoErrors."""
25
26
27 def git(cmd):
28 """Execute a git command with the given list as arguments. Return stdout."""
29 proc = subprocess.Popen(['git'] + cmd, cwd=os.environ['FREEDICT_TOOLS'],
30 stdout=subprocess.PIPE)
31 stdout = proc.communicate()[0].strip()
32 ret = proc.wait()
33 if ret:
34 raise ReleaseError("`git %s` exited with exit code %d" % \
35 (' '.join(cmd), ret))
36 return stdout.decode(sys.getdefaultencoding())
37
38 def get_tools_release():
39 """Retrieve the latest FreeDicttools release as a tuple with containing
40 (version, date, downloadlink)."""
41 if not 'FREEDICT_TOOLS' in os.environ or not shutil.which('git'):
42 raise ReleaseError(("Unable to retrieve list of releases of "
43 "FreeDict tools. Either FREEDICT_TOOLS is unset or git not "
44 "installed."))
45 releases = git(['tag']).split('\n')
46 max_ver = '0.0.0'
47 for tag in releases:
48 max_ver = semver.max_ver(max_ver, tag)
49 if max_ver == '0.0.0':
50 raise ReleaseError("No tools releases found.")
51 date = re.search('^([0-9]{4}-[0-9]{2}-[0-9]{2})',
52 git(['show', '-s', '--pretty=format:%ci'])).groups()[0]
53 return (max_ver, date, '{}/freedict-tools-{}.tar.xz'.format(
54 RELEASE_HTTP_TOOL_BASE, max_ver))
55
56 def get_release_info_for_dict(path, version):
57 """Retrieve information about the releases of a dictionary."""
58 files = {}
59 name = None
60 version = normalize_version(version)
61 for file in os.listdir(path):
62 format = DownloadFormat.get_type(file)
63 if not format:
64 continue # ignore unknown file naming, possibly outdated or unsupported formats
65
66 parsed_name, file_version_str = format.value.search(file).groups()
67 file_version = normalize_version(file_version_str)
68
69 if file_version != version:
70 raise ReleaseError('Version from file name "%s" did not match version of directory "%s"' \
71 % (file, version))
72 if not name:
73 name = parsed_name
74 elif parsed_name != name:
75 raise ReleaseError('Found two different dictionary identifiers in %s: %s and %s' \
76 % (path, parsed_name, name))
77 full_path = os.path.join(path, file)
78 try:
79 with open(full_path + '.sha512', 'r') as f:
80 sha = f.read().strip().split(' ')[0]
81 except FileNotFoundError:
82 raise FileNotFoundError('expected a sha512 checksum, found nothing',
83 full_path + '.sha512', 'r')
84 files[full_path] = (name, format, sha)
85 if not files:
86 raise ReleaseError("no downloads available for %s" % path)
87 return files
88
89
90 def get_all_downloads(root):
91 """Get all paths to a downloadable file from a given root.
92 This function walks the given path and returns all paths to files with the
93 relative to the given root."""
94 # only match directories with two iso 639-3 codes, separated by "-"
95 dirpattern = re.compile(r'[a-z]{3}-[a-z]{3}')
96 release_directories = tuple(os.path.join(root, e) for e in os.listdir(root)
97 if os.path.isdir(os.path.join(root, e)) and dirpattern.search(e) \
98 and not 'tools' in e.lower())
99 if not release_directories:
100 raise ReleaseError("%s: no released dictionary found" % root)
101
102 dictionaries_with_release = {} # collect dictionaries which have a release
103 for dictdir in release_directories:
104 # iterate over subdirectories which represent versions
105 for version_dir in (os.path.join(dictdir, f) for f in os.listdir(dictdir)):
106 version = os.path.basename(version_dir)
107
108 try:
109 files = get_release_info_for_dict(version_dir, version)
110 except ReleaseError as e:
111 if e.args[0].startswith('no downloads available'):
112 continue # skip versions with no or unknown releases
113 else:
114 raise
115 if not files:
116 print('Warning: no files in "%s", skipping...' % version)
117 continue
118 name = next(iter(files.values()))[0]
119 if not name in dictionaries_with_release:
120 dictionaries_with_release[name] = {}
121 # transform {path: (name, format, hash)} to (path, format, hash)
122 dictionaries_with_release[name][version] = tuple((k, v[1], v[2])
123 for k,v in files.items())
124 return dictionaries_with_release
125
126
127 def get_latest_version(release_information):
128 """Iterate over object and return the latest version, as defined by
•129 semver."""
130 latest = None
131 latest_strict = None # might contain '-' replaced through '.'
132 for version in release_information:
133 try:
•134 version_strict = normalize_version(version)
135 except ValueError as e:
136 raise ReleaseError(e.args)
137 if not latest:
138 latest = version
139 latest_strict = version_strict
140 else:
141 if version_strict > latest_strict:
142 latest = version
143 latest_strict = version_strict
144 if not latest:
145 raise ReleaseError("No versions found for " % repr(release_information))
146 return latest
147
148 def github_request(path):
149 """Make a GitHub API request and return aJSON object."""
150 url = "https://api.github.com/{}".format(path)
151 request_headers = {'User-Agent': USER_AGENT}
152 request = urllib.request.Request(url, headers=request_headers)
153 with urllib.request.urlopen(request) as f:
154 return json.loads(f.read().decode('UTF-8'))
155
156 def get_latest_tools_release():
157 latest = {'name': '0.0.0'}
158 for tag in github_request("repos/freedict/{}/tags".format(TOOLS_REPO)):
159 latest = max(latest, tag, key=lambda t: t['name'])
160 if latest['name'] == '0.0.0':
161 raise ValueError("could not find a release for FreeDict tools")
162 commit_url = latest['commit']['url']
163 api_suffix = commit_url[commit_url.find('.com/') + 5:]
164 commit_meta = github_request(api_suffix)['commit']['committer']
165 # validate format
166 date = datetime.strptime(commit_meta['date'].split('T')[0], '%Y-%m-%d')
167 date = date.strftime('%Y-%m-%d')
168 # We don't use the API-provided download URL (latest['tarball']), as it
169 # contains a funny number in the unpacked directory.
170 github_uri = f"https://github.com/freedict/tools/archive/refs/tags/{latest['name']}.tar.gz"
171 request = urllib.request.Request(github_uri,
172 headers={'User-Agent': USER_AGENT})
173 with urllib.request.urlopen(request) as f:
174 checksum = hashlib.sha512(f.read()).hexdigest()
175 return {'version': latest['name'],
176 'date': date,
177 'URL': github_uri,
178 'checksum': checksum}