| 1 |
#!/usr/bin/env python |
| 2 |
# License: GPLv3 Copyright: 2014, Kovid Goyal <kovid at kovidgoyal.net> |
| 3 |
|
| 4 |
import codecs |
| 5 |
import glob |
| 6 |
import os |
| 7 |
import re |
| 8 |
import sys |
| 9 |
import tempfile |
| 10 |
|
| 11 |
from lxml import etree |
| 12 |
|
| 13 |
from calibre import browser |
| 14 |
from calibre.constants import config_dir |
| 15 |
from calibre.utils.resources import get_path as P |
| 16 |
from calibre.utils.xml_parse import safe_xml_fromstring |
| 17 |
from calibre.utils.zipfile import ZipFile |
| 18 |
|
| 19 |
NS_MAP = { |
| 20 |
'oor': 'http://openoffice.org/2001/registry', |
| 21 |
'xs': 'http://www.w3.org/2001/XMLSchema', |
| 22 |
'manifest': 'http://openoffice.org/2001/manifest', |
| 23 |
} |
| 24 |
|
| 25 |
|
| 26 |
def XPath(x): |
| 27 |
return etree.XPath(x, namespaces=NS_MAP) |
| 28 |
|
| 29 |
|
| 30 |
BUILTIN_LOCALES = {'en-US', 'en-GB', 'es-ES'} |
| 31 |
ONLINE_DICTIONARY_BASE_URL = 'https://raw.githubusercontent.com/LibreOffice/dictionaries/master/' |
| 32 |
|
| 33 |
|
| 34 |
def parse_xcu(raw, origin='%origin%'): |
| 35 |
"Get the dictionary and affix file names as well as supported locales for each dictionary" |
| 36 |
ans = {} |
| 37 |
root = safe_xml_fromstring(raw) |
| 38 |
|
| 39 |
for node in XPath('//prop[@oor:name="Format"]/value[text()="DICT_SPELL"]/../..')(root): |
| 40 |
value = XPath('descendant::prop[@oor:name="Locations"]/value')(node) |
| 41 |
if len(value[0]) == 0: |
| 42 |
# The value node has no children, use its text |
| 43 |
paths = ''.join(XPath('descendant::prop[@oor:name="Locations"]/value/text()')(node)).replace('%origin%', origin).split() |
| 44 |
else: |
| 45 |
# Use the text of the value nodes children |
| 46 |
paths = [c.text.replace('%origin%', origin) for v in value for c in v.iterchildren('*') if c.text] |
| 47 |
aff, dic = paths if paths[0].endswith('.aff') else reversed(paths) |
| 48 |
locales = ''.join(XPath('descendant::prop[@oor:name="Locales"]/value/text()')(node)).split() |
| 49 |
if not locales: |
| 50 |
locales = [str(item) for item in XPath('descendant::prop[@oor:name="Locales"]/value/it/text()')(node)] |
| 51 |
ans[(dic, aff)] = locales |
| 52 |
return ans |
| 53 |
|
| 54 |
|
| 55 |
def convert_to_utf8(dic_data, aff_data, errors='strict'): |
| 56 |
m = re.search(rb'^SET\s+(\S+)$', aff_data[:2048], flags=re.MULTILINE) |
| 57 |
if m is not None: |
| 58 |
enc = m.group(1).decode('ascii', 'replace') |
| 59 |
if enc.upper() not in ('UTF-8', 'UTF8'): |
| 60 |
try: |
| 61 |
codecs.lookup(enc) |
| 62 |
except LookupError: |
| 63 |
pass |
| 64 |
else: |
| 65 |
aff_data = aff_data[: m.start()] + b'SET UTF-8' + aff_data[m.end() :] |
| 66 |
aff_data = aff_data.decode(enc, errors).encode('utf-8') |
| 67 |
dic_data = dic_data.decode(enc, errors).encode('utf-8') |
| 68 |
return dic_data, aff_data |
| 69 |
|
| 70 |
|
| 71 |
def import_from_libreoffice_source_tree(source_path): |
| 72 |
dictionaries = {} |
| 73 |
for x in glob.glob(os.path.join(source_path, '*', 'dictionaries.xcu')): |
| 74 |
origin = os.path.dirname(x) |
| 75 |
with open(x, 'rb') as f: |
| 76 |
dictionaries.update(parse_xcu(f.read(), origin)) |
| 77 |
|
| 78 |
base = P('dictionaries', allow_user_override=False) |
| 79 |
want_locales = set(BUILTIN_LOCALES) |
| 80 |
|
| 81 |
for (dic, aff), locales in dictionaries.items(): |
| 82 |
c = set(locales) & want_locales |
| 83 |
if c: |
| 84 |
locale = tuple(c)[0] |
| 85 |
want_locales.discard(locale) |
| 86 |
dest = os.path.join(base, locale) |
| 87 |
if not os.path.exists(dest): |
| 88 |
os.makedirs(dest) |
| 89 |
with open(dic, 'rb') as df, open(aff, 'rb') as af: |
| 90 |
dd, ad = convert_to_utf8(df.read(), af.read()) |
| 91 |
for src, raw in ((dic, dd), (aff, ad)): |
| 92 |
with open(os.path.join(dest, locale + os.path.splitext(src)[1]), 'wb') as df: |
| 93 |
df.write(raw) |
| 94 |
with open(os.path.join(dest, 'locales'), 'wb') as f: |
| 95 |
locales.sort(key=lambda x: (0, x) if x == locale else (1, x)) |
| 96 |
f.write(('\n'.join(locales)).encode('utf-8')) |
| 97 |
|
| 98 |
if want_locales: |
| 99 |
raise Exception(f'Failed to find dictionaries for some wanted locales: {want_locales}') |
| 100 |
|
| 101 |
|
| 102 |
def fill_country_code(x): |
| 103 |
return {'lt': 'lt_LT'}.get(x, x) |
| 104 |
|
| 105 |
|
| 106 |
def uniq(vals, kmap=lambda x: x): |
| 107 |
"""Remove all duplicates from vals, while preserving order. kmap must be a |
| 108 |
callable that returns a hashable value for every item in vals""" |
| 109 |
vals = vals or () |
| 110 |
lvals = (kmap(x) for x in vals) |
| 111 |
seen = set() |
| 112 |
seen_add = seen.add |
| 113 |
return tuple(x for x, k in zip(vals, lvals) if k not in seen and not seen_add(k)) # ty: ignore[redundant-condition] |
| 114 |
|
| 115 |
|
| 116 |
def _import_from_virtual_directory(read_file_func, name, dest_dir=None, prefix='dic-'): |
| 117 |
from calibre.spell.dictionary import parse_lang_code |
| 118 |
|
| 119 |
dest_dir = dest_dir or os.path.join(config_dir, 'dictionaries') |
| 120 |
if not os.path.exists(dest_dir): |
| 121 |
os.makedirs(dest_dir) |
| 122 |
num = 0 |
| 123 |
|
| 124 |
root = safe_xml_fromstring(read_file_func('META-INF/manifest.xml')) |
| 125 |
xcu = XPath('//manifest:file-entry[@manifest:media-type="application/vnd.sun.star.configuration-data"]')(root)[0].get( |
| 126 |
'{{{}}}full-path'.format(NS_MAP['manifest']) |
| 127 |
) |
| 128 |
for (dic, aff), locales in parse_xcu(read_file_func(xcu), origin='').items(): |
| 129 |
dic, aff = dic.lstrip('/'), aff.lstrip('/') |
| 130 |
d = tempfile.mkdtemp(prefix=prefix, dir=dest_dir) |
| 131 |
locales = uniq([x for x in map(fill_country_code, locales) if parse_lang_code(x).countrycode]) |
| 132 |
if not locales: |
| 133 |
continue |
| 134 |
metadata = [name] + list(locales) |
| 135 |
with open(os.path.join(d, 'locales'), 'wb') as f: |
| 136 |
f.write(('\n'.join(metadata)).encode('utf-8')) |
| 137 |
dd, ad = convert_to_utf8(read_file_func(dic), read_file_func(aff)) |
| 138 |
with open(os.path.join(d, f'{locales[0]}.dic'), 'wb') as f: |
| 139 |
f.write(dd) |
| 140 |
with open(os.path.join(d, f'{locales[0]}.aff'), 'wb') as f: |
| 141 |
f.write(ad) |
| 142 |
num += 1 |
| 143 |
return num |
| 144 |
|
| 145 |
|
| 146 |
def import_from_oxt(source_path, name, dest_dir=None, prefix='dic-'): |
| 147 |
with ZipFile(source_path) as zf: |
| 148 |
|
| 149 |
def read_file(key): |
| 150 |
try: |
| 151 |
return zf.open(key).read() |
| 152 |
except KeyError: |
| 153 |
# Some dictionaries apparently put the xcu in a sub-directory |
| 154 |
# and incorrectly make paths relative to that directory instead |
| 155 |
# of the root, for example: |
| 156 |
# http://extensions.libreoffice.org/extension-center/italian-dictionary-thesaurus-hyphenation-patterns/releases/4.1/dict-it.oxt |
| 157 |
while key.startswith('../'): |
| 158 |
key = key[3:] |
| 159 |
return zf.open(key.lstrip('/')).read() |
| 160 |
|
| 161 |
return _import_from_virtual_directory(read_file, name, dest_dir=dest_dir, prefix=prefix) |
| 162 |
|
| 163 |
|
| 164 |
def import_from_online(directory, name, dest_dir=None, prefix='dic-'): |
| 165 |
br = browser(timeout=30) |
| 166 |
|
| 167 |
def read_file(key): |
| 168 |
try: |
| 169 |
rp = br.open('/'.join((ONLINE_DICTIONARY_BASE_URL, directory, key))) |
| 170 |
return rp.read() |
| 171 |
except Exception as err: |
| 172 |
if getattr(err, 'code', -1) != 404: |
| 173 |
raise |
| 174 |
# Some dictionaries apparently put the dic and aff file in a |
| 175 |
# sub-directory dictionaries and incorrectly make paths relative |
| 176 |
# to that directory instead of the root, for example: |
| 177 |
# https://github.com/LibreOffice/dictionaries/tree/master/ca |
| 178 |
rp = br.open('/'.join((ONLINE_DICTIONARY_BASE_URL, directory, 'dictionaries', key))) |
| 179 |
return rp.read() |
| 180 |
|
| 181 |
return _import_from_virtual_directory(read_file, name, dest_dir=dest_dir, prefix=prefix) |
| 182 |
|
| 183 |
|
| 184 |
if __name__ == '__main__': |
| 185 |
import_from_libreoffice_source_tree(sys.argv[-1]) |