Debian Codegraph @

src/calibre/spell/import_from.py

raw
1 #!/usr/bin/env python
2 # License: GPLv3 Copyright: 2014, Kovid Goyal <kovid at kovidgoyal.net>
3
4 import codecs
5 import glob
6 import os
7 import re
8 import sys
9 import tempfile
10
11 from lxml import etree
12
13 from calibre import browser
14 from calibre.constants import config_dir
15 from calibre.utils.resources import get_path as P
16 from calibre.utils.xml_parse import safe_xml_fromstring
17 from calibre.utils.zipfile import ZipFile
18
19 NS_MAP = {
20 'oor': 'http://openoffice.org/2001/registry',
21 'xs': 'http://www.w3.org/2001/XMLSchema',
22 'manifest': 'http://openoffice.org/2001/manifest',
23 }
24
25
26 def XPath(x):
27 return etree.XPath(x, namespaces=NS_MAP)
28
29
30 BUILTIN_LOCALES = {'en-US', 'en-GB', 'es-ES'}
31 ONLINE_DICTIONARY_BASE_URL = 'https://raw.githubusercontent.com/LibreOffice/dictionaries/master/'
32
33
34 def parse_xcu(raw, origin='%origin%'):
35 "Get the dictionary and affix file names as well as supported locales for each dictionary"
36 ans = {}
37 root = safe_xml_fromstring(raw)
38
39 for node in XPath('//prop[@oor:name="Format"]/value[text()="DICT_SPELL"]/../..')(root):
40 value = XPath('descendant::prop[@oor:name="Locations"]/value')(node)
41 if len(value[0]) == 0:
42 # The value node has no children, use its text
43 paths = ''.join(XPath('descendant::prop[@oor:name="Locations"]/value/text()')(node)).replace('%origin%', origin).split()
44 else:
45 # Use the text of the value nodes children
46 paths = [c.text.replace('%origin%', origin) for v in value for c in v.iterchildren('*') if c.text]
47 aff, dic = paths if paths[0].endswith('.aff') else reversed(paths)
48 locales = ''.join(XPath('descendant::prop[@oor:name="Locales"]/value/text()')(node)).split()
49 if not locales:
50 locales = [str(item) for item in XPath('descendant::prop[@oor:name="Locales"]/value/it/text()')(node)]
51 ans[(dic, aff)] = locales
52 return ans
53
54
55 def convert_to_utf8(dic_data, aff_data, errors='strict'):
56 m = re.search(rb'^SET\s+(\S+)$', aff_data[:2048], flags=re.MULTILINE)
57 if m is not None:
58 enc = m.group(1).decode('ascii', 'replace')
59 if enc.upper() not in ('UTF-8', 'UTF8'):
60 try:
61 codecs.lookup(enc)
62 except LookupError:
63 pass
64 else:
65 aff_data = aff_data[: m.start()] + b'SET UTF-8' + aff_data[m.end() :]
66 aff_data = aff_data.decode(enc, errors).encode('utf-8')
67 dic_data = dic_data.decode(enc, errors).encode('utf-8')
68 return dic_data, aff_data
69
70
71 def import_from_libreoffice_source_tree(source_path):
72 dictionaries = {}
73 for x in glob.glob(os.path.join(source_path, '*', 'dictionaries.xcu')):
74 origin = os.path.dirname(x)
75 with open(x, 'rb') as f:
76 dictionaries.update(parse_xcu(f.read(), origin))
77
78 base = P('dictionaries', allow_user_override=False)
79 want_locales = set(BUILTIN_LOCALES)
80
81 for (dic, aff), locales in dictionaries.items():
82 c = set(locales) & want_locales
83 if c:
84 locale = tuple(c)[0]
85 want_locales.discard(locale)
86 dest = os.path.join(base, locale)
87 if not os.path.exists(dest):
88 os.makedirs(dest)
89 with open(dic, 'rb') as df, open(aff, 'rb') as af:
90 dd, ad = convert_to_utf8(df.read(), af.read())
91 for src, raw in ((dic, dd), (aff, ad)):
92 with open(os.path.join(dest, locale + os.path.splitext(src)[1]), 'wb') as df:
93 df.write(raw)
94 with open(os.path.join(dest, 'locales'), 'wb') as f:
95 locales.sort(key=lambda x: (0, x) if x == locale else (1, x))
96 f.write(('\n'.join(locales)).encode('utf-8'))
97
98 if want_locales:
99 raise Exception(f'Failed to find dictionaries for some wanted locales: {want_locales}')
100
101
102 def fill_country_code(x):
103 return {'lt': 'lt_LT'}.get(x, x)
104
105
106 def uniq(vals, kmap=lambda x: x):
107 """Remove all duplicates from vals, while preserving order. kmap must be a
108 callable that returns a hashable value for every item in vals"""
109 vals = vals or ()
110 lvals = (kmap(x) for x in vals)
111 seen = set()
112 seen_add = seen.add
113 return tuple(x for x, k in zip(vals, lvals) if k not in seen and not seen_add(k)) # ty: ignore[redundant-condition]
114
115
116 def _import_from_virtual_directory(read_file_func, name, dest_dir=None, prefix='dic-'):
117 from calibre.spell.dictionary import parse_lang_code
118
119 dest_dir = dest_dir or os.path.join(config_dir, 'dictionaries')
120 if not os.path.exists(dest_dir):
121 os.makedirs(dest_dir)
122 num = 0
123
124 root = safe_xml_fromstring(read_file_func('META-INF/manifest.xml'))
125 xcu = XPath('//manifest:file-entry[@manifest:media-type="application/vnd.sun.star.configuration-data"]')(root)[0].get(
126 '{{{}}}full-path'.format(NS_MAP['manifest'])
127 )
128 for (dic, aff), locales in parse_xcu(read_file_func(xcu), origin='').items():
129 dic, aff = dic.lstrip('/'), aff.lstrip('/')
130 d = tempfile.mkdtemp(prefix=prefix, dir=dest_dir)
131 locales = uniq([x for x in map(fill_country_code, locales) if parse_lang_code(x).countrycode])
132 if not locales:
133 continue
134 metadata = [name] + list(locales)
135 with open(os.path.join(d, 'locales'), 'wb') as f:
136 f.write(('\n'.join(metadata)).encode('utf-8'))
137 dd, ad = convert_to_utf8(read_file_func(dic), read_file_func(aff))
138 with open(os.path.join(d, f'{locales[0]}.dic'), 'wb') as f:
139 f.write(dd)
140 with open(os.path.join(d, f'{locales[0]}.aff'), 'wb') as f:
141 f.write(ad)
142 num += 1
143 return num
144
145
146 def import_from_oxt(source_path, name, dest_dir=None, prefix='dic-'):
147 with ZipFile(source_path) as zf:
148
149 def read_file(key):
150 try:
151 return zf.open(key).read()
152 except KeyError:
153 # Some dictionaries apparently put the xcu in a sub-directory
154 # and incorrectly make paths relative to that directory instead
155 # of the root, for example:
156 # http://extensions.libreoffice.org/extension-center/italian-dictionary-thesaurus-hyphenation-patterns/releases/4.1/dict-it.oxt
157 while key.startswith('../'):
158 key = key[3:]
159 return zf.open(key.lstrip('/')).read()
160
161 return _import_from_virtual_directory(read_file, name, dest_dir=dest_dir, prefix=prefix)
162
163
164 def import_from_online(directory, name, dest_dir=None, prefix='dic-'):
165 br = browser(timeout=30)
166
167 def read_file(key):
168 try:
169 rp = br.open('/'.join((ONLINE_DICTIONARY_BASE_URL, directory, key)))
170 return rp.read()
171 except Exception as err:
172 if getattr(err, 'code', -1) != 404:
173 raise
174 # Some dictionaries apparently put the dic and aff file in a
175 # sub-directory dictionaries and incorrectly make paths relative
176 # to that directory instead of the root, for example:
177 # https://github.com/LibreOffice/dictionaries/tree/master/ca
178 rp = br.open('/'.join((ONLINE_DICTIONARY_BASE_URL, directory, 'dictionaries', key)))
179 return rp.read()
180
181 return _import_from_virtual_directory(read_file, name, dest_dir=dest_dir, prefix=prefix)
182
183
184 if __name__ == '__main__':
185 import_from_libreoffice_source_tree(sys.argv[-1])