| 1 |
#!/usr/bin/env python |
| 2 |
# License: GPLv3 Copyright: 2019, Kovid Goyal <kovid at kovidgoyal.net> |
| 3 |
|
| 4 |
import glob |
| 5 |
import hashlib |
| 6 |
import json |
| 7 |
import os |
| 8 |
import shutil |
| 9 |
import tarfile |
| 10 |
from io import BytesIO |
| 11 |
|
| 12 |
from setup.revendor import ReVendor |
| 13 |
|
| 14 |
|
| 15 |
def locales_from_dicts(dicts): |
| 16 |
ans = {} |
| 17 |
for path in dicts: |
| 18 |
name = bname = os.path.basename(path) |
| 19 |
name = name[len('hyph_') : -len('.dic')] |
| 20 |
ans[name.replace('-', '_')] = bname |
| 21 |
return ans |
| 22 |
|
| 23 |
|
| 24 |
def locales_from_xcu(xcu, dicts): |
| 25 |
from lxml import etree |
| 26 |
|
| 27 |
with open(xcu, 'rb') as f: |
| 28 |
root = etree.fromstring(f.read(), parser=etree.XMLParser(recover=True, no_network=True, resolve_entities=False)) |
| 29 |
ans = {} |
| 30 |
dicts = {os.path.basename(x) for x in dicts} |
| 31 |
for value in root.xpath('//*[contains(text(),"DICT_HYPH")]'): |
| 32 |
node = value.getparent().getparent() |
| 33 |
locales = path = None |
| 34 |
for prop in node: |
| 35 |
name = prop.get('{http://openoffice.org/2001/registry}name') |
| 36 |
if name == 'Locales': |
| 37 |
locales = [x.replace('-', '_') for x in prop[0].text.split()] |
| 38 |
elif name == 'Locations': |
| 39 |
path = prop[0].text.strip().split('/')[-1] |
| 40 |
if locales and path in dicts: |
| 41 |
for locale in locales: |
| 42 |
ans[locale] = path |
| 43 |
return ans |
| 44 |
|
| 45 |
|
| 46 |
def process_dictionaries(src, output_dir): |
| 47 |
locale_data = {} |
| 48 |
for x in os.listdir(src): |
| 49 |
q = os.path.join(src, x) |
| 50 |
if not os.path.isdir(q): |
| 51 |
continue |
| 52 |
dicts = tuple(glob.glob(os.path.join(q, 'hyph_*.dic'))) |
| 53 |
if not dicts: |
| 54 |
continue |
| 55 |
xcu = os.path.join(q, 'dictionaries.xcu') |
| 56 |
locales = locales_from_xcu(xcu, dicts) if os.path.exists(xcu) else locales_from_dicts(dicts) |
| 57 |
if locales: |
| 58 |
locale_data.update(locales) |
| 59 |
for d in dicts: |
| 60 |
shutil.copyfile(d, os.path.join(output_dir, os.path.basename(d))) |
| 61 |
data = json.dumps(locale_data, indent=2) |
| 62 |
if not isinstance(data, bytes): |
| 63 |
data = data.encode('utf-8') |
| 64 |
with open(os.path.join(output_dir, 'locales.json'), 'wb') as f: |
| 65 |
f.write(data) |
| 66 |
|
| 67 |
|
| 68 |
def compress_tar(buf, outf): |
| 69 |
buf.seek(0) |
| 70 |
import lzma |
| 71 |
|
| 72 |
outf.write(lzma.compress(buf.getvalue(), preset=9 | lzma.PRESET_EXTREME)) |
| 73 |
|
| 74 |
|
| 75 |
class Hyphenation(ReVendor): |
| 76 |
description = 'Download the hyphenation dictionaries' |
| 77 |
NAME = 'hyphenation' |
| 78 |
TAR_NAME = 'hyphenation dictionaries' |
| 79 |
VERSION = 'master' |
| 80 |
DOWNLOAD_URL = f'https://github.com/LibreOffice/dictionaries/archive/{VERSION}.tar.gz' |
| 81 |
CAN_USE_SYSTEM_VERSION = False |
| 82 |
|
| 83 |
def run(self, opts): |
| 84 |
self.clean() |
| 85 |
os.makedirs(self.vendored_dir) |
| 86 |
with self.temp_dir() as dl_src, self.temp_dir() as output_dir: |
| 87 |
src = opts.path_to_hyphenation or self.download_vendor_release(dl_src, opts.hyphenation_url) |
| 88 |
process_dictionaries(src, output_dir) |
| 89 |
dics = sorted(x for x in os.listdir(output_dir) if x.endswith('.dic')) |
| 90 |
m = hashlib.sha1() |
| 91 |
for dic in dics: |
| 92 |
with open(os.path.join(output_dir, dic), 'rb') as f: |
| 93 |
m.update(f.read()) |
| 94 |
hsh = str(m.hexdigest()) |
| 95 |
buf = BytesIO() |
| 96 |
with tarfile.TarFile(fileobj=buf, mode='w') as tf: |
| 97 |
for dic in dics: |
| 98 |
with open(os.path.join(output_dir, dic), 'rb') as df: |
| 99 |
tinfo = tf.gettarinfo(arcname=dic, fileobj=df) |
| 100 |
tinfo.mtime = 0 |
| 101 |
tinfo.uid = tinfo.gid = 1000 |
| 102 |
tinfo.uname = tinfo.gname = 'kovid' |
| 103 |
tf.addfile(tinfo, df) |
| 104 |
with open(os.path.join(self.vendored_dir, 'dictionaries.tar.xz'), 'wb') as f: |
| 105 |
compress_tar(buf, f) |
| 106 |
with open(os.path.join(self.vendored_dir, 'sha1sum'), 'w') as f: |
| 107 |
f.write(hsh) |
| 108 |
shutil.copy(os.path.join(output_dir, 'locales.json'), self.vendored_dir) |