Debian Codegraph @

setup/hyphenation.py

raw
1 #!/usr/bin/env python
2 # License: GPLv3 Copyright: 2019, Kovid Goyal <kovid at kovidgoyal.net>
3
4 import glob
5 import hashlib
6 import json
7 import os
8 import shutil
9 import tarfile
10 from io import BytesIO
11
12 from setup.revendor import ReVendor
13
14
15 def locales_from_dicts(dicts):
16 ans = {}
17 for path in dicts:
18 name = bname = os.path.basename(path)
19 name = name[len('hyph_') : -len('.dic')]
20 ans[name.replace('-', '_')] = bname
21 return ans
22
23
24 def locales_from_xcu(xcu, dicts):
25 from lxml import etree
26
27 with open(xcu, 'rb') as f:
28 root = etree.fromstring(f.read(), parser=etree.XMLParser(recover=True, no_network=True, resolve_entities=False))
29 ans = {}
30 dicts = {os.path.basename(x) for x in dicts}
31 for value in root.xpath('//*[contains(text(),"DICT_HYPH")]'):
32 node = value.getparent().getparent()
33 locales = path = None
34 for prop in node:
35 name = prop.get('{http://openoffice.org/2001/registry}name')
36 if name == 'Locales':
37 locales = [x.replace('-', '_') for x in prop[0].text.split()]
38 elif name == 'Locations':
39 path = prop[0].text.strip().split('/')[-1]
40 if locales and path in dicts:
41 for locale in locales:
42 ans[locale] = path
43 return ans
44
45
46 def process_dictionaries(src, output_dir):
47 locale_data = {}
48 for x in os.listdir(src):
49 q = os.path.join(src, x)
50 if not os.path.isdir(q):
51 continue
52 dicts = tuple(glob.glob(os.path.join(q, 'hyph_*.dic')))
53 if not dicts:
54 continue
55 xcu = os.path.join(q, 'dictionaries.xcu')
56 locales = locales_from_xcu(xcu, dicts) if os.path.exists(xcu) else locales_from_dicts(dicts)
57 if locales:
58 locale_data.update(locales)
59 for d in dicts:
60 shutil.copyfile(d, os.path.join(output_dir, os.path.basename(d)))
61 data = json.dumps(locale_data, indent=2)
62 if not isinstance(data, bytes):
63 data = data.encode('utf-8')
64 with open(os.path.join(output_dir, 'locales.json'), 'wb') as f:
65 f.write(data)
66
67
68 def compress_tar(buf, outf):
69 buf.seek(0)
70 import lzma
71
72 outf.write(lzma.compress(buf.getvalue(), preset=9 | lzma.PRESET_EXTREME))
73
74
75 class Hyphenation(ReVendor):
76 description = 'Download the hyphenation dictionaries'
77 NAME = 'hyphenation'
78 TAR_NAME = 'hyphenation dictionaries'
79 VERSION = 'master'
80 DOWNLOAD_URL = f'https://github.com/LibreOffice/dictionaries/archive/{VERSION}.tar.gz'
81 CAN_USE_SYSTEM_VERSION = False
82
83 def run(self, opts):
84 self.clean()
85 os.makedirs(self.vendored_dir)
86 with self.temp_dir() as dl_src, self.temp_dir() as output_dir:
87 src = opts.path_to_hyphenation or self.download_vendor_release(dl_src, opts.hyphenation_url)
88 process_dictionaries(src, output_dir)
89 dics = sorted(x for x in os.listdir(output_dir) if x.endswith('.dic'))
90 m = hashlib.sha1()
91 for dic in dics:
92 with open(os.path.join(output_dir, dic), 'rb') as f:
93 m.update(f.read())
94 hsh = str(m.hexdigest())
95 buf = BytesIO()
96 with tarfile.TarFile(fileobj=buf, mode='w') as tf:
97 for dic in dics:
98 with open(os.path.join(output_dir, dic), 'rb') as df:
99 tinfo = tf.gettarinfo(arcname=dic, fileobj=df)
100 tinfo.mtime = 0
101 tinfo.uid = tinfo.gid = 1000
102 tinfo.uname = tinfo.gname = 'kovid'
103 tf.addfile(tinfo, df)
104 with open(os.path.join(self.vendored_dir, 'dictionaries.tar.xz'), 'wb') as f:
105 compress_tar(buf, f)
106 with open(os.path.join(self.vendored_dir, 'sha1sum'), 'w') as f:
107 f.write(hsh)
108 shutil.copy(os.path.join(output_dir, 'locales.json'), self.vendored_dir)