| 1 |
"""Collect all information about released dictionaries.""" |
| 2 |
|
| 3 |
from datetime import datetime |
| 4 |
import hashlib |
| 5 |
import json |
| 6 |
import os |
| 7 |
import re |
| 8 |
import shutil |
| 9 |
import subprocess |
| 10 |
import sys |
| 11 |
import urllib.request |
| 12 |
|
| 13 |
import semver |
| 14 |
|
| 15 |
from .config import RELEASE_HTTP_TOOL_BASE |
| 16 |
from .dictionary import DownloadFormat, normalize_version |
| 17 |
|
| 18 |
# relative to github.com/freedict/ |
| 19 |
TOOLS_REPO = 'tools' |
| 20 |
USER_AGENT = 'Friendly FreeDict Helper' |
| 21 |
|
| 22 |
class ReleaseError(Exception): |
| 23 |
"""This error can occur, when information about a release is gathered. It |
| 24 |
wraps all sorts of ValueErrors and IoErrors.""" |
| 25 |
|
| 26 |
|
| 27 |
def git(cmd): |
| 28 |
"""Execute a git command with the given list as arguments. Return stdout.""" |
| 29 |
proc = subprocess.Popen(['git'] + cmd, cwd=os.environ['FREEDICT_TOOLS'], |
| 30 |
stdout=subprocess.PIPE) |
| 31 |
stdout = proc.communicate()[0].strip() |
| 32 |
ret = proc.wait() |
| 33 |
if ret: |
| 34 |
raise ReleaseError("`git %s` exited with exit code %d" % \ |
| 35 |
(' '.join(cmd), ret)) |
| 36 |
return stdout.decode(sys.getdefaultencoding()) |
| 37 |
|
| 38 |
def get_tools_release(): |
| 39 |
"""Retrieve the latest FreeDicttools release as a tuple with containing |
| 40 |
(version, date, downloadlink).""" |
| 41 |
if not 'FREEDICT_TOOLS' in os.environ or not shutil.which('git'): |
| 42 |
raise ReleaseError(("Unable to retrieve list of releases of " |
| 43 |
"FreeDict tools. Either FREEDICT_TOOLS is unset or git not " |
| 44 |
"installed.")) |
| 45 |
releases = git(['tag']).split('\n') |
| 46 |
max_ver = '0.0.0' |
| 47 |
for tag in releases: |
| 48 |
max_ver = semver.max_ver(max_ver, tag) |
| 49 |
if max_ver == '0.0.0': |
| 50 |
raise ReleaseError("No tools releases found.") |
| 51 |
date = re.search('^([0-9]{4}-[0-9]{2}-[0-9]{2})', |
| 52 |
git(['show', '-s', '--pretty=format:%ci'])).groups()[0] |
| 53 |
return (max_ver, date, '{}/freedict-tools-{}.tar.xz'.format( |
| 54 |
RELEASE_HTTP_TOOL_BASE, max_ver)) |
| 55 |
|
| 56 |
def get_release_info_for_dict(path, version): |
| 57 |
"""Retrieve information about the releases of a dictionary.""" |
| 58 |
files = {} |
| 59 |
name = None |
| 60 |
version = normalize_version(version) |
| 61 |
for file in os.listdir(path): |
| 62 |
format = DownloadFormat.get_type(file) |
| 63 |
if not format: |
| 64 |
continue # ignore unknown file naming, possibly outdated or unsupported formats |
| 65 |
|
| 66 |
parsed_name, file_version_str = format.value.search(file).groups() |
| 67 |
file_version = normalize_version(file_version_str) |
| 68 |
|
| 69 |
if file_version != version: |
| 70 |
raise ReleaseError('Version from file name "%s" did not match version of directory "%s"' \ |
| 71 |
% (file, version)) |
| 72 |
if not name: |
| 73 |
name = parsed_name |
| 74 |
elif parsed_name != name: |
| 75 |
raise ReleaseError('Found two different dictionary identifiers in %s: %s and %s' \ |
| 76 |
% (path, parsed_name, name)) |
| 77 |
full_path = os.path.join(path, file) |
| 78 |
try: |
| 79 |
with open(full_path + '.sha512', 'r') as f: |
| 80 |
sha = f.read().strip().split(' ')[0] |
| 81 |
except FileNotFoundError: |
| 82 |
raise FileNotFoundError('expected a sha512 checksum, found nothing', |
| 83 |
full_path + '.sha512', 'r') |
| 84 |
files[full_path] = (name, format, sha) |
| 85 |
if not files: |
| 86 |
raise ReleaseError("no downloads available for %s" % path) |
| 87 |
return files |
| 88 |
|
| 89 |
|
| 90 |
def get_all_downloads(root): |
| 91 |
"""Get all paths to a downloadable file from a given root. |
| 92 |
This function walks the given path and returns all paths to files with the |
| 93 |
relative to the given root.""" |
| 94 |
# only match directories with two iso 639-3 codes, separated by "-" |
| 95 |
dirpattern = re.compile(r'[a-z]{3}-[a-z]{3}') |
| 96 |
release_directories = tuple(os.path.join(root, e) for e in os.listdir(root) |
| 97 |
if os.path.isdir(os.path.join(root, e)) and dirpattern.search(e) \ |
| 98 |
and not 'tools' in e.lower()) |
| 99 |
if not release_directories: |
| 100 |
raise ReleaseError("%s: no released dictionary found" % root) |
| 101 |
|
| 102 |
dictionaries_with_release = {} # collect dictionaries which have a release |
| 103 |
for dictdir in release_directories: |
| 104 |
# iterate over subdirectories which represent versions |
| 105 |
for version_dir in (os.path.join(dictdir, f) for f in os.listdir(dictdir)): |
| 106 |
version = os.path.basename(version_dir) |
| 107 |
|
| 108 |
try: |
| 109 |
files = get_release_info_for_dict(version_dir, version) |
| 110 |
except ReleaseError as e: |
| 111 |
if e.args[0].startswith('no downloads available'): |
| 112 |
continue # skip versions with no or unknown releases |
| 113 |
else: |
| 114 |
raise |
| 115 |
if not files: |
| 116 |
print('Warning: no files in "%s", skipping...' % version) |
| 117 |
continue |
| 118 |
name = next(iter(files.values()))[0] |
| 119 |
if not name in dictionaries_with_release: |
| 120 |
dictionaries_with_release[name] = {} |
| 121 |
# transform {path: (name, format, hash)} to (path, format, hash) |
| 122 |
dictionaries_with_release[name][version] = tuple((k, v[1], v[2]) |
| 123 |
for k,v in files.items()) |
| 124 |
return dictionaries_with_release |
| 125 |
|
| 126 |
|
| 127 |
def get_latest_version(release_information): |
| 128 |
"""Iterate over object and return the latest version, as defined by |
| •129 |
semver.""" |
| 130 |
latest = None |
| 131 |
latest_strict = None # might contain '-' replaced through '.' |
| 132 |
for version in release_information: |
| 133 |
try: |
| •134 |
version_strict = normalize_version(version) |
| 135 |
except ValueError as e: |
| 136 |
raise ReleaseError(e.args) |
| 137 |
if not latest: |
| 138 |
latest = version |
| 139 |
latest_strict = version_strict |
| 140 |
else: |
| 141 |
if version_strict > latest_strict: |
| 142 |
latest = version |
| 143 |
latest_strict = version_strict |
| 144 |
if not latest: |
| 145 |
raise ReleaseError("No versions found for " % repr(release_information)) |
| 146 |
return latest |
| 147 |
|
| 148 |
def github_request(path): |
| 149 |
"""Make a GitHub API request and return aJSON object.""" |
| 150 |
url = "https://api.github.com/{}".format(path) |
| 151 |
request_headers = {'User-Agent': USER_AGENT} |
| 152 |
request = urllib.request.Request(url, headers=request_headers) |
| 153 |
with urllib.request.urlopen(request) as f: |
| 154 |
return json.loads(f.read().decode('UTF-8')) |
| 155 |
|
| 156 |
def get_latest_tools_release(): |
| 157 |
latest = {'name': '0.0.0'} |
| 158 |
for tag in github_request("repos/freedict/{}/tags".format(TOOLS_REPO)): |
| 159 |
latest = max(latest, tag, key=lambda t: t['name']) |
| 160 |
if latest['name'] == '0.0.0': |
| 161 |
raise ValueError("could not find a release for FreeDict tools") |
| 162 |
commit_url = latest['commit']['url'] |
| 163 |
api_suffix = commit_url[commit_url.find('.com/') + 5:] |
| 164 |
commit_meta = github_request(api_suffix)['commit']['committer'] |
| 165 |
# validate format |
| 166 |
date = datetime.strptime(commit_meta['date'].split('T')[0], '%Y-%m-%d') |
| 167 |
date = date.strftime('%Y-%m-%d') |
| 168 |
# We don't use the API-provided download URL (latest['tarball']), as it |
| 169 |
# contains a funny number in the unpacked directory. |
| 170 |
github_uri = f"https://github.com/freedict/tools/archive/refs/tags/{latest['name']}.tar.gz" |
| 171 |
request = urllib.request.Request(github_uri, |
| 172 |
headers={'User-Agent': USER_AGENT}) |
| 173 |
with urllib.request.urlopen(request) as f: |
| 174 |
checksum = hashlib.sha512(f.read()).hexdigest() |
| 175 |
return {'version': latest['name'], |
| 176 |
'date': date, |
| 177 |
'URL': github_uri, |
| 178 |
'checksum': checksum} |