| 1 |
#!/usr/bin/env python |
| 2 |
|
| 3 |
|
| 4 |
__license__ = 'GPL v3' |
| 5 |
__copyright__ = '2009, Kovid Goyal <[email protected]>' |
| 6 |
__docformat__ = 'restructuredtext en' |
| 7 |
|
| 8 |
import errno |
| 9 |
import glob |
| 10 |
import hashlib |
| 11 |
import inspect |
| 12 |
import json |
| 13 |
import os |
| 14 |
import re |
| 15 |
import shlex |
| 16 |
import shutil |
| 17 |
import subprocess |
| 18 |
import sys |
| 19 |
import tempfile |
| 20 |
import textwrap |
| 21 |
import time |
| 22 |
from collections import defaultdict |
| 23 |
from functools import lru_cache, partial |
| 24 |
from locale import normalize as normalize_locale |
| 25 |
|
| 26 |
from setup import Command, __appname__, __version__, build_cache_dir, dump_json, edit_file, is_ci, require_git_master |
| 27 |
from setup.iso_codes import iso_data |
| 28 |
from setup.parallel_build import batched_parallel_jobs |
| 29 |
|
| 30 |
|
| 31 |
def serialize_msgid(text): |
| 32 |
"""Serialize a string in the format used by msgid in GNU POT files.""" |
| 33 |
if not text: |
| 34 |
return 'msgid ""\n' |
| 35 |
# Escape backslashes and quotes |
| 36 |
escaped = text.replace('\\', r'\\').replace('"', r'\"') |
| 37 |
ans = ['msgid ""'] |
| 38 |
lines = escaped.splitlines() |
| 39 |
for line in lines: |
| 40 |
trailer = '"' if line is lines[-1] else r'\n"' |
| 41 |
ans.append(f'"{line}{trailer}') |
| 42 |
return '\n'.join(ans) |
| 43 |
|
| 44 |
|
| 45 |
def qt_sources(): |
| 46 |
qtdir = os.environ.get('QT_SRC', '/usr/src/qt6/qtbase') |
| 47 |
j = partial(os.path.join, qtdir) |
| 48 |
return list( |
| 49 |
map( |
| 50 |
j, |
| 51 |
[ |
| 52 |
'src/gui/kernel/qplatformtheme.cpp', |
| 53 |
'src/widgets/dialogs/qcolordialog.cpp', |
| 54 |
'src/widgets/dialogs/qfontdialog.cpp', |
| 55 |
'src/widgets/widgets/qscrollbar.cpp', |
| 56 |
], |
| 57 |
) |
| 58 |
) |
| 59 |
|
| 60 |
|
| 61 |
@lru_cache(maxsize=2) |
| 62 |
def tx_exe(): |
| 63 |
return os.environ.get('TX', shutil.which('tx-cli') or shutil.which('tx') or 'tx') |
| 64 |
|
| 65 |
|
| 66 |
class POT(Command): # {{{ |
| 67 |
description = 'Update the .pot translation template and upload it' |
| 68 |
TRANSLATIONS = os.path.join(os.path.dirname(Command.SRC), 'translations') |
| 69 |
MANUAL = os.path.join(os.path.dirname(Command.SRC), 'manual') |
| 70 |
|
| 71 |
def tx(self, cmd, **kw): |
| 72 |
kw['cwd'] = kw.get('cwd', self.TRANSLATIONS) |
| 73 |
if hasattr(cmd, 'format'): |
| 74 |
cmd = shlex.split(cmd) |
| 75 |
cmd = [tx_exe()] + cmd |
| 76 |
self.info(' '.join(cmd)) |
| 77 |
return subprocess.check_call(cmd, **kw) |
| 78 |
|
| 79 |
def git(self, cmd, **kw): |
| 80 |
kw['cwd'] = kw.get('cwd', self.TRANSLATIONS) |
| 81 |
if hasattr(cmd, 'format'): |
| 82 |
cmd = shlex.split(cmd) |
| 83 |
f = getattr(subprocess, ('call' if kw.pop('use_call', False) else 'check_call')) |
| 84 |
return f(['git'] + cmd, **kw) |
| 85 |
|
| 86 |
def upload_pot(self, resource): |
| 87 |
self.tx(['push', '-r', 'calibre.' + resource, '-s'], cwd=self.TRANSLATIONS) |
| 88 |
|
| 89 |
def source_files(self): |
| 90 |
ans = [self.a(self.j(self.MANUAL, x)) for x in ('custom.py', 'conf.py')] |
| 91 |
for root, _, files in os.walk(self.j(self.SRC, __appname__)): |
| 92 |
for name in files: |
| 93 |
if name.endswith('.py'): |
| 94 |
ans.append(self.a(self.j(root, name))) |
| 95 |
return ans |
| 96 |
|
| 97 |
def get_ffml_docs(self): |
| 98 |
from calibre.gui2.dialogs.template_general_info import ffml_doc, general_doc |
| 99 |
from calibre.utils.formatter_functions import formatter_functions |
| 100 |
|
| 101 |
all_funcs = formatter_functions().get_builtins() |
| 102 |
ans = [] |
| 103 |
for ff in all_funcs.values(): |
| 104 |
lnum = inspect.getsourcelines(ff.__doc__getter__)[1] |
| 105 |
text = ff.__doc__getter__().msgid |
| 106 |
ans.append(f'#: src/calibre/utils/formatter_function.py:{lnum}\n' + serialize_msgid(text) + '\nmsgstr ""\n\n') |
| 107 |
for ff in (ffml_doc, general_doc): |
| 108 |
lnum = inspect.getsourcelines(ff)[1] |
| 109 |
text = ff().msgid |
| 110 |
ans.append(f'#: src/calibre/gui2/dialogs/template_general_info.py:{lnum}\n' + serialize_msgid(text) + '\nmsgstr ""\n\n') |
| 111 |
return ''.join(ans) |
| 112 |
|
| 113 |
def get_tweaks_docs(self): |
| 114 |
path = self.a(self.j(self.SRC, '..', 'resources', 'default_tweaks.py')) |
| 115 |
with open(path, 'rb') as f: |
| 116 |
raw = f.read().decode('utf-8') |
| 117 |
msgs = [] |
| 118 |
lines = list(raw.splitlines()) |
| 119 |
for i, line in enumerate(lines): |
| 120 |
if line.startswith('#:'): |
| 121 |
msgs.append((i, line[2:].strip())) |
| 122 |
j = i |
| 123 |
block = [] |
| 124 |
while True: |
| 125 |
j += 1 |
| 126 |
line = lines[j] |
| 127 |
if not line.startswith('#'): |
| 128 |
break |
| 129 |
block.append(line[1:].strip()) |
| 130 |
if block: |
| 131 |
msgs.append((i + 1, '\n'.join(block))) |
| 132 |
|
| 133 |
ans = [] |
| 134 |
for lineno, msg in msgs: |
| 135 |
ans.append(f'#: {path}:{lineno}') |
| 136 |
slash = chr(92) |
| 137 |
msg = msg.replace(slash, slash * 2).replace('"', r'\"').replace('\n', r'\n').replace('\r', r'\r').replace('\t', r'\t') |
| 138 |
ans.append(f'msgid "{msg}"') |
| 139 |
ans.append('msgstr ""') |
| 140 |
ans.append('') |
| 141 |
|
| 142 |
return '\n'.join(ans) |
| 143 |
|
| 144 |
def get_content_server_strings(self): |
| 145 |
self.info('Generating translation template for content_server') |
| 146 |
from calibre import walk |
| 147 |
from calibre.utils.rapydscript import create_pot |
| 148 |
|
| 149 |
files = (f for f in walk(self.j(self.SRC, 'pyj')) if f.endswith('.pyj')) |
| 150 |
pottext = create_pot(files).encode('utf-8') |
| 151 |
dest = self.j(self.TRANSLATIONS, 'content-server', 'content-server.pot') |
| 152 |
with open(dest, 'wb') as f: |
| 153 |
f.write(pottext) |
| 154 |
self.upload_pot(resource='content_server') |
| 155 |
self.git(['add', dest]) |
| 156 |
|
| 157 |
def get_user_manual_docs(self): |
| 158 |
self.info('Generating translation templates for user_manual') |
| 159 |
base = tempfile.mkdtemp() |
| 160 |
subprocess.check_call([sys.executable, self.j(self.d(self.SRC), 'manual', 'build.py'), 'gettext', base]) |
| 161 |
tbase = self.j(self.TRANSLATIONS, 'manual') |
| 162 |
for x in os.listdir(base): |
| 163 |
if not x.endswith('.pot'): |
| 164 |
continue |
| 165 |
src, dest = self.j(base, x), self.j(tbase, x) |
| 166 |
needs_import = not os.path.exists(dest) |
| 167 |
with open(src, 'rb') as s, open(dest, 'wb') as d: |
| 168 |
shutil.copyfileobj(s, d) |
| 169 |
bname = os.path.splitext(x)[0] |
| 170 |
slug = 'user_manual_' + bname |
| 171 |
if needs_import: |
| 172 |
self.tx(['set', '-r', 'calibre.' + slug, '--source', '-l', 'en', '-t', 'PO', dest]) |
| 173 |
with open(self.j(self.d(tbase), '.tx/config'), 'r+b') as f: |
| 174 |
lines = f.read().decode('utf-8').splitlines() |
| 175 |
for i in range(len(lines)): |
| 176 |
line = lines[i].strip() |
| 177 |
if line == f'[calibre.{slug}]': |
| 178 |
lines.insert(i + 1, f'file_filter = manual/<lang>/{bname}.po') |
| 179 |
f.seek(0), f.truncate(), f.write('\n'.join(lines).encode('utf-8')) |
| 180 |
break |
| 181 |
else: |
| 182 |
raise SystemExit(f'Failed to add file_filter for slug={slug} to config file') |
| 183 |
self.git('add .tx/config') |
| 184 |
self.upload_pot(resource=slug) |
| 185 |
self.git(['add', dest]) |
| 186 |
shutil.rmtree(base) |
| 187 |
|
| 188 |
def get_website_strings(self): |
| 189 |
self.info('Generating translation template for website') |
| 190 |
self.wn_path = os.path.expanduser('~/work/srv/main/static/generate.py') |
| 191 |
data = subprocess.check_output([self.wn_path, '--pot', '/tmp/wn']) |
| 192 |
data = json.loads(data) |
| 193 |
|
| 194 |
def do(name): |
| 195 |
messages = data[name] |
| 196 |
bdir = os.path.join(self.TRANSLATIONS, name) |
| 197 |
if not os.path.exists(bdir): |
| 198 |
os.makedirs(bdir) |
| 199 |
pot = os.path.abspath(os.path.join(bdir, name + '.pot')) |
| 200 |
with open(pot, 'wb') as f: |
| 201 |
f.write(self.pot_header().encode('utf-8')) |
| 202 |
f.write(b'\n') |
| 203 |
f.write('\n'.join(messages).encode('utf-8')) |
| 204 |
self.upload_pot(resource=name) |
| 205 |
self.git(['add', pot]) |
| 206 |
|
| 207 |
do('website') |
| 208 |
do('changelog') |
| 209 |
|
| 210 |
def pot_header(self, appname=__appname__, version=__version__): |
| 211 |
return textwrap.dedent('''\ |
| 212 |
# Translation template file.. |
| 213 |
# Copyright (C) {year} Kovid Goyal |
| 214 |
# Kovid Goyal <[email protected]>, {year}. |
| 215 |
# |
| 216 |
msgid "" |
| 217 |
msgstr "" |
| 218 |
"Project-Id-Version: {appname} {version}\\n" |
| 219 |
"POT-Creation-Date: {time}\\n" |
| 220 |
"PO-Revision-Date: {time}\\n" |
| 221 |
"Last-Translator: Automatically generated\\n" |
| 222 |
"Language-Team: LANGUAGE\\n" |
| 223 |
"MIME-Version: 1.0\\n" |
| 224 |
"Report-Msgid-Bugs-To: https://bugs.launchpad.net/calibre\\n" |
| 225 |
"Plural-Forms: nplurals=INTEGER; plural=EXPRESSION;\\n" |
| 226 |
"Content-Type: text/plain; charset=UTF-8\\n" |
| 227 |
"Content-Transfer-Encoding: 8bit\\n" |
| 228 |
|
| 229 |
''').format(appname=appname, version=version, year=time.strftime('%Y'), time=time.strftime('%Y-%m-%d %H:%M+%Z')) |
| 230 |
|
| 231 |
def run(self, opts): |
| 232 |
if not is_ci: |
| 233 |
require_git_master() |
| 234 |
if not is_ci: |
| 235 |
self.get_website_strings() |
| 236 |
self.get_content_server_strings() |
| 237 |
self.get_user_manual_docs() |
| 238 |
files = self.source_files() |
| 239 |
qt_inputs = qt_sources() |
| 240 |
pot_header = self.pot_header() |
| 241 |
|
| 242 |
with tempfile.NamedTemporaryFile() as fl: |
| 243 |
fl.write('\n'.join(files).encode('utf-8')) |
| 244 |
fl.flush() |
| 245 |
out = tempfile.NamedTemporaryFile(suffix='.pot', delete=False) |
| 246 |
out.close() |
| 247 |
self.info('Creating translations template...') |
| 248 |
subprocess.check_call([ |
| 249 |
'xgettext', |
| 250 |
'-f', |
| 251 |
fl.name, |
| 252 |
'--default-domain=calibre', |
| 253 |
'-o', |
| 254 |
out.name, |
| 255 |
'-L', |
| 256 |
'Python', |
| 257 |
'--from-code=UTF-8', |
| 258 |
'--sort-by-file', |
| 259 |
'--omit-header', |
| 260 |
'--no-wrap', |
| 261 |
'-k__', |
| 262 |
'-kpgettext:1c,2', |
| 263 |
'--add-comments=NOTE:', |
| 264 |
]) |
| 265 |
subprocess.check_call( |
| 266 |
[ |
| 267 |
'xgettext', |
| 268 |
'-j', |
| 269 |
'--default-domain=calibre', |
| 270 |
'-o', |
| 271 |
out.name, |
| 272 |
'--from-code=UTF-8', |
| 273 |
'--sort-by-file', |
| 274 |
'--omit-header', |
| 275 |
'--no-wrap', |
| 276 |
'-kQT_TRANSLATE_NOOP:2', |
| 277 |
'-ktr', |
| 278 |
'-ktranslate:2', |
| 279 |
] |
| 280 |
+ qt_inputs |
| 281 |
) |
| 282 |
with open(out.name, 'rb') as f: |
| 283 |
src = f.read().decode('utf-8') |
| 284 |
os.remove(out.name) |
| 285 |
src = pot_header + '\n' + src |
| 286 |
src += '\n\n' + self.get_tweaks_docs() |
| 287 |
src += '\n\n' + self.get_ffml_docs() |
| 288 |
bdir = os.path.join(self.TRANSLATIONS, __appname__) |
| 289 |
if not os.path.exists(bdir): |
| 290 |
os.makedirs(bdir) |
| 291 |
pot = os.path.join(bdir, 'main.pot') |
| 292 |
# Workaround for bug in xgettext: |
| 293 |
# https://savannah.gnu.org/bugs/index.php?41668 |
| 294 |
src = re.sub(r'#, python-brace-format\s+msgid ""\s+.*<code>{0:</code>', lambda m: m.group().replace('python-brace', 'no-python-brace'), src) |
| 295 |
with open(pot, 'wb') as f: |
| 296 |
f.write(src.encode('utf-8')) |
| 297 |
self.info('Translations template:', os.path.abspath(pot)) |
| 298 |
self.upload_pot(resource='main') |
| 299 |
self.git(['add', os.path.abspath(pot)]) |
| 300 |
|
| 301 |
if not is_ci and self.git('diff-index --cached --quiet --ignore-submodules HEAD --', use_call=True) != 0: |
| 302 |
self.git(['commit', '-m', 'Updated translation templates']) |
| 303 |
self.git('push') |
| 304 |
|
| 305 |
return pot |
| 306 |
|
| 307 |
|
| 308 |
# }}} |
| 309 |
|
| 310 |
|
| 311 |
class Translations(POT): # {{{ |
| 312 |
description = '''Compile the translations''' |
| 313 |
DEST = os.path.join(os.path.dirname(POT.SRC), 'resources', 'localization', 'locales') |
| 314 |
|
| 315 |
@property |
| 316 |
def cache_dir(self): |
| 317 |
ans = self.j(build_cache_dir(), 'translations') |
| 318 |
if not hasattr(self, 'cache_dir_created'): |
| 319 |
self.cache_dir_created = True |
| 320 |
try: |
| 321 |
os.mkdir(ans) |
| 322 |
except OSError as err: |
| 323 |
if err.errno != errno.EEXIST: |
| 324 |
raise |
| 325 |
return ans |
| 326 |
|
| 327 |
def cache_name(self, f): |
| 328 |
f = os.path.relpath(f, self.d(self.SRC)) |
| 329 |
return f.replace(os.sep, '.').replace('/', '.').lstrip('.') |
| 330 |
|
| 331 |
def read_cache(self, f): |
| 332 |
cname = self.cache_name(f) |
| 333 |
try: |
| 334 |
with open(self.j(self.cache_dir, cname), 'rb') as f: |
| 335 |
data = f.read() |
| 336 |
return data[:20], data[20:] |
| 337 |
except OSError as err: |
| 338 |
if err.errno != errno.ENOENT: |
| 339 |
raise |
| 340 |
return None, None |
| 341 |
|
| 342 |
def write_cache(self, data, h, f): |
| 343 |
cname = self.cache_name(f) |
| 344 |
assert len(h) == 20 |
| 345 |
with open(self.j(self.cache_dir, cname), 'wb') as f: |
| 346 |
f.write(h), f.write(data) |
| 347 |
|
| 348 |
def is_po_file_ok(self, x): |
| 349 |
bname = os.path.splitext(os.path.basename(x))[0] |
| 350 |
# [email protected] is identical to sr.po. And we don't support country |
| 351 |
# specific variants except for a few. |
| 352 |
if '_' in bname: |
| 353 |
return bname.partition('_')[0] in ('pt', 'zh', 'bn') |
| 354 |
return bname != 'sr@latin' |
| 355 |
|
| 356 |
def po_files(self): |
| 357 |
return [x for x in glob.glob(os.path.join(self.TRANSLATIONS, __appname__, '*.po')) if self.is_po_file_ok(x)] |
| 358 |
|
| 359 |
def mo_file(self, po_file): |
| 360 |
locale = os.path.splitext(os.path.basename(po_file))[0] |
| 361 |
return locale, os.path.join(self.DEST, locale, 'messages.mo') |
| 362 |
|
| 363 |
def run(self, opts): |
| 364 |
self.compile_main_translations() |
| 365 |
self.compile_content_server_translations() |
| 366 |
self.freeze_locales() |
| 367 |
self.compile_user_manual_translations() |
| 368 |
self.compile_website_translations() |
| 369 |
self.compile_changelog_translations() |
| 370 |
|
| 371 |
def compile_group(self, files, handle_stats=None, action_per_file=None, make_translated_strings_unique=False, keyfunc=lambda x: x): |
| 372 |
ok_files = [] |
| 373 |
hashmap = {} |
| 374 |
|
| 375 |
def stats_cache(src, data=None): |
| 376 |
cname = self.cache_name(keyfunc(src)) + '.stats.json' |
| 377 |
with open(self.j(self.cache_dir, cname), ('rb' if data is None else 'wb')) as f: |
| 378 |
if data is None: |
| 379 |
return json.loads(f.read()) |
| 380 |
data = json.dumps(data) |
| 381 |
if not isinstance(data, bytes): |
| 382 |
data = data.encode('utf-8') |
| 383 |
f.write(data) |
| 384 |
|
| 385 |
for src, dest in files: |
| 386 |
base = os.path.dirname(dest) |
| 387 |
if not os.path.exists(base): |
| 388 |
os.makedirs(base) |
| 389 |
data, h = self.hash_and_data(src, keyfunc) |
| 390 |
current_hash = h.digest() |
| 391 |
saved_hash, saved_data = self.read_cache(keyfunc(src)) |
| 392 |
if current_hash == saved_hash: |
| 393 |
with open(dest, 'wb') as d: |
| 394 |
d.write(saved_data) |
| 395 |
if handle_stats is not None: |
| 396 |
handle_stats(src, stats_cache(src)) |
| 397 |
else: |
| 398 |
ok_files.append((src, dest)) |
| 399 |
hashmap[keyfunc(src)] = current_hash |
| 400 |
if action_per_file is not None: |
| 401 |
action_per_file(src) |
| 402 |
|
| 403 |
self.info(f'\tCompiling {len(ok_files)} files') |
| 404 |
items = [] |
| 405 |
results = batched_parallel_jobs( |
| 406 |
[sys.executable, self.j(self.SRC, 'calibre', 'translations', 'msgfmt.py'), 'STDIN', 'uniqify' if make_translated_strings_unique else ' '], ok_files |
| 407 |
) |
| 408 |
for (src, dest), data in zip(ok_files, results): |
| 409 |
items.append((src, dest, data)) |
| 410 |
|
| 411 |
for src, dest, data in items: |
| 412 |
self.write_cache(open(dest, 'rb').read(), hashmap[keyfunc(src)], keyfunc(src)) |
| 413 |
stats_cache(src, data) |
| 414 |
if handle_stats is not None: |
| 415 |
handle_stats(src, data) |
| 416 |
|
| 417 |
def compile_main_translations(self): |
| 418 |
l = {} |
| 419 |
lc_dataf = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'lc_data.py') |
| 420 |
exec(compile(open(lc_dataf, 'rb').read(), lc_dataf, 'exec'), l, l) |
| 421 |
lcdata = {k: dict(v) for k, v in l['data']} |
| 422 |
self.info('Compiling main UI translation files...') |
| 423 |
fmap = {f: self.mo_file(f) for f in self.po_files()} |
| 424 |
files = [(f, fmap[f][1]) for f in self.po_files()] |
| 425 |
|
| 426 |
def action_per_file(f): |
| 427 |
locale, dest = fmap[f] |
| 428 |
ln = normalize_locale(locale).partition('.')[0] |
| 429 |
if ln in lcdata: |
| 430 |
ld = lcdata[ln] |
| 431 |
lcdest = self.j(self.d(dest), 'lcdata.calibre_msgpack') |
| 432 |
from calibre.utils.serialize import msgpack_dumps |
| 433 |
|
| 434 |
with open(lcdest, 'wb') as lcf: |
| 435 |
lcf.write(msgpack_dumps(ld)) |
| 436 |
|
| 437 |
stats = {} |
| 438 |
|
| 439 |
def handle_stats(f, data): |
| 440 |
trans, untrans = data['translated'], data['untranslated'] |
| 441 |
total = trans + untrans |
| 442 |
locale = fmap[f][0] |
| 443 |
stats[locale] = min(1.0, float(trans) / total) |
| 444 |
|
| 445 |
self.compile_group(files, handle_stats=handle_stats, action_per_file=action_per_file) |
| 446 |
|
| 447 |
self.info('Compiling ISO639 files...') |
| 448 |
files = [] |
| 449 |
skip_iso = { |
| 450 |
'si', |
| 451 |
'te', |
| 452 |
'km', |
| 453 |
'en_GB', |
| 454 |
'en_AU', |
| 455 |
'en_CA', |
| 456 |
'yi', |
| 457 |
'ku', |
| 458 |
'my', |
| 459 |
'uz@Latn', |
| 460 |
'fil', |
| 461 |
'hy', |
| 462 |
'ltg', |
| 463 |
'km_KH', |
| 464 |
'km', |
| 465 |
'ur', |
| 466 |
'ml', |
| 467 |
'fo', |
| 468 |
'ug', |
| 469 |
'jv', |
| 470 |
'nds', |
| 471 |
} |
| 472 |
|
| 473 |
def handle_stats(f, data): |
| 474 |
if False and data['uniqified']: |
| 475 |
print(f'{data["uniqified"]:3d} non-unique language name translations in {os.path.basename(f)}', file=sys.stderr) |
| 476 |
|
| 477 |
with tempfile.TemporaryDirectory() as tdir: |
| 478 |
iso_data.extract_po_files('iso_639-3', tdir) |
| 479 |
for f, (locale, dest) in fmap.items(): |
| 480 |
iscpo = {'zh_HK': 'zh_CN'}.get(locale, locale) |
| 481 |
iso639 = self.j(tdir, f'{iscpo}.po') |
| 482 |
if os.path.exists(iso639): |
| 483 |
files.append((iso639, self.j(self.d(dest), 'iso639.mo'))) |
| 484 |
else: |
| 485 |
iscpo = iscpo.partition('_')[0] |
| 486 |
iso639 = self.j(tdir, f'{iscpo}.po') |
| 487 |
if os.path.exists(iso639): |
| 488 |
files.append((iso639, self.j(self.d(dest), 'iso639.mo'))) |
| 489 |
elif locale not in skip_iso: |
| 490 |
self.warn('No ISO 639 translations for locale:', locale) |
| 491 |
self.compile_group( |
| 492 |
files, |
| 493 |
make_translated_strings_unique=True, |
| 494 |
handle_stats=handle_stats, |
| 495 |
keyfunc=lambda x: os.path.join(self.d(self.SRC), 'iso639', os.path.basename(x)), |
| 496 |
) |
| 497 |
|
| 498 |
self.info('Compiling ISO3166 files...') |
| 499 |
files = [] |
| 500 |
skip_iso = {'en_GB', 'en_AU', 'en_CA', 'yi', 'ku', 'uz@Latn', 'ltg', 'nds', 'jv'} |
| 501 |
with tempfile.TemporaryDirectory() as tdir: |
| 502 |
iso_data.extract_po_files('iso_3166-1', tdir) |
| 503 |
for f, (locale, dest) in fmap.items(): |
| 504 |
pofile = self.j(tdir, f'{locale}.po') |
| 505 |
if os.path.exists(pofile): |
| 506 |
files.append((pofile, self.j(self.d(dest), 'iso3166.mo'))) |
| 507 |
else: |
| 508 |
pofile = self.j(tdir, f'{locale.partition("_")[0]}.po') |
| 509 |
if os.path.exists(pofile): |
| 510 |
files.append((pofile, self.j(self.d(dest), 'iso3166.mo'))) |
| 511 |
elif locale not in skip_iso: |
| 512 |
self.warn('No ISO 3166 translations for locale:', locale) |
| 513 |
self.compile_group( |
| 514 |
files, |
| 515 |
make_translated_strings_unique=True, |
| 516 |
handle_stats=lambda f, d: None, |
| 517 |
keyfunc=lambda x: os.path.join(self.d(self.SRC), 'iso3166', os.path.basename(x)), |
| 518 |
) |
| 519 |
|
| 520 |
dest = self.stats |
| 521 |
base = self.d(dest) |
| 522 |
try: |
| 523 |
os.mkdir(base) |
| 524 |
except OSError as err: |
| 525 |
if err.errno != errno.EEXIST: |
| 526 |
raise |
| 527 |
from calibre.utils.serialize import msgpack_dumps |
| 528 |
|
| 529 |
with open(dest, 'wb') as f: |
| 530 |
f.write(msgpack_dumps(stats)) |
| 531 |
|
| 532 |
def hash_and_data(self, f, keyfunc=lambda x: x): |
| 533 |
with open(f, 'rb') as s: |
| 534 |
data = s.read() |
| 535 |
h = hashlib.sha1(data) |
| 536 |
h.update(keyfunc(f).encode('utf-8')) |
| 537 |
return data, h |
| 538 |
|
| 539 |
def compile_content_server_translations(self): |
| 540 |
self.info('Compiling content-server translations') |
| 541 |
from calibre.utils.rapydscript import msgfmt |
| 542 |
from calibre.utils.zipfile import ZIP_DEFLATED, ZIP_STORED, ZipFile, ZipInfo |
| 543 |
|
| 544 |
with ZipFile(self.j(self.RESOURCES, 'content-server', 'locales.zip'), 'w', ZIP_DEFLATED) as zf: |
| 545 |
for src in glob.glob(os.path.join(self.TRANSLATIONS, 'content-server', '*.po')): |
| 546 |
if not self.is_po_file_ok(src): |
| 547 |
continue |
| 548 |
data, h = self.hash_and_data(src) |
| 549 |
current_hash = h.digest() |
| 550 |
saved_hash, saved_data = self.read_cache(src) |
| 551 |
if current_hash == saved_hash: |
| 552 |
raw = saved_data |
| 553 |
else: |
| 554 |
# self.info('\tParsing ' + os.path.basename(src)) |
| 555 |
raw = None |
| 556 |
po_data = data.decode('utf-8') |
| 557 |
data = json.loads(msgfmt(po_data)) |
| 558 |
translated_entries = {k: v for k, v in data['entries'].items() if v and sum(map(len, v))} |
| 559 |
data['entries'] = translated_entries |
| 560 |
data['hash'] = h.hexdigest() |
| 561 |
cdata = b'{}' |
| 562 |
if translated_entries: |
| 563 |
raw = json.dumps(data, ensure_ascii=False, sort_keys=True) |
| 564 |
if isinstance(raw, str): |
| 565 |
raw = raw.encode('utf-8') |
| 566 |
cdata = raw |
| 567 |
self.write_cache(cdata, current_hash, src) |
| 568 |
if raw: |
| 569 |
zi = ZipInfo(os.path.basename(src).rpartition('.')[0]) |
| 570 |
zi.compress_type = ZIP_STORED if is_ci else ZIP_DEFLATED |
| 571 |
zf.writestr(zi, raw) |
| 572 |
|
| 573 |
def freeze_locales(self): |
| 574 |
zf = self.DEST + '.zip' |
| 575 |
from calibre import CurrentDir |
| 576 |
from calibre.utils.zipfile import ZIP_DEFLATED, ZipFile |
| 577 |
|
| 578 |
with ZipFile(zf, 'w', ZIP_DEFLATED) as zf: |
| 579 |
with CurrentDir(self.DEST): |
| 580 |
zf.add_dir('.') |
| 581 |
shutil.rmtree(self.DEST) |
| 582 |
|
| 583 |
@property |
| 584 |
def stats(self): |
| 585 |
return self.j(self.d(self.DEST), 'stats.calibre_msgpack') |
| 586 |
|
| 587 |
def _compile_website_translations(self, name='website', threshold=50): |
| 588 |
from calibre.ptempfile import TemporaryDirectory |
| 589 |
from calibre.utils.localization import get_language, translator_for_lang |
| 590 |
from calibre.utils.zipfile import ZIP_STORED, ZipFile, ZipInfo |
| 591 |
|
| 592 |
self.info('Compiling', name, 'translations...') |
| 593 |
srcbase = self.j(self.d(self.SRC), 'translations', name) |
| 594 |
if not os.path.exists(srcbase): |
| 595 |
os.makedirs(srcbase) |
| 596 |
fmap = {} |
| 597 |
files = [] |
| 598 |
stats = {} |
| 599 |
done = [] |
| 600 |
|
| 601 |
def handle_stats(src, data): |
| 602 |
locale = fmap[src] |
| 603 |
trans, untrans = data['translated'], data['untranslated'] |
| 604 |
total = trans + untrans |
| 605 |
stats[locale] = round(100 * trans / total) |
| 606 |
|
| 607 |
with TemporaryDirectory() as tdir, ZipFile(self.j(srcbase, 'locales.zip'), 'w', ZIP_STORED) as zf: |
| 608 |
for f in os.listdir(srcbase): |
| 609 |
if f.endswith('.po'): |
| 610 |
if not self.is_po_file_ok(f): |
| 611 |
continue |
| 612 |
l = f.partition('.')[0] |
| 613 |
pf = l.split('_')[0] |
| 614 |
if pf == 'en': |
| 615 |
continue |
| 616 |
d = os.path.join(tdir, l + '.mo') |
| 617 |
f = os.path.join(srcbase, f) |
| 618 |
fmap[f] = l |
| 619 |
files.append((f, d)) |
| 620 |
self.compile_group(files, handle_stats=handle_stats) |
| 621 |
|
| 622 |
for locale, translated in stats.items(): |
| 623 |
if translated >= threshold: |
| 624 |
with open(os.path.join(tdir, locale + '.mo'), 'rb') as f: |
| 625 |
raw = f.read() |
| 626 |
zi = ZipInfo(os.path.basename(f.name)) |
| 627 |
zi.compress_type = ZIP_STORED |
| 628 |
zf.writestr(zi, raw) |
| 629 |
done.append(locale) |
| 630 |
dl = done + ['en'] |
| 631 |
|
| 632 |
lang_names = {} |
| 633 |
for l in dl: |
| 634 |
translator = translator_for_lang(l)['translator'] |
| 635 |
t = translator.gettext |
| 636 |
t = partial(get_language, gettext_func=t) |
| 637 |
lang_names[l] = {x: t(x) for x in dl} |
| 638 |
zi = ZipInfo('lang-names.json') |
| 639 |
zi.compress_type = ZIP_STORED |
| 640 |
zf.writestr(zi, json.dumps(lang_names, ensure_ascii=False).encode('utf-8')) |
| 641 |
return done |
| 642 |
|
| 643 |
def compile_website_translations(self): |
| 644 |
done = self._compile_website_translations() |
| 645 |
dest = self.j(self.d(self.stats), 'website-languages.txt') |
| 646 |
data = ' '.join(sorted(done)) |
| 647 |
if not isinstance(data, bytes): |
| 648 |
data = data.encode('utf-8') |
| 649 |
with open(dest, 'wb') as f: |
| 650 |
f.write(data) |
| 651 |
|
| 652 |
def compile_changelog_translations(self): |
| 653 |
self._compile_website_translations('changelog', threshold=0) |
| 654 |
|
| 655 |
def compile_user_manual_translations(self): |
| 656 |
self.info('Compiling user manual translations...') |
| 657 |
srcbase = self.j(self.d(self.SRC), 'translations', 'manual') |
| 658 |
destbase = self.j(self.d(self.SRC), 'manual', 'locale') |
| 659 |
complete = {} |
| 660 |
all_stats = defaultdict(lambda: {'translated': 0, 'untranslated': 0}) |
| 661 |
files = [] |
| 662 |
for x in os.listdir(srcbase): |
| 663 |
q = self.j(srcbase, x) |
| 664 |
if not os.path.isdir(q) or not self.is_po_file_ok(q): |
| 665 |
continue |
| 666 |
dest = self.j(destbase, x, 'LC_MESSAGES') |
| 667 |
if os.path.exists(dest): |
| 668 |
shutil.rmtree(dest) |
| 669 |
os.makedirs(dest) |
| 670 |
for po in os.listdir(q): |
| 671 |
if not po.endswith('.po'): |
| 672 |
continue |
| 673 |
mofile = self.j(dest, po.rpartition('.')[0] + '.mo') |
| 674 |
files.append((self.j(q, po), mofile)) |
| 675 |
|
| 676 |
def handle_stats(src, data): |
| 677 |
locale = self.b(self.d(src)) |
| 678 |
stats = all_stats[locale] |
| 679 |
stats['translated'] += data['translated'] |
| 680 |
stats['untranslated'] += data['untranslated'] |
| 681 |
|
| 682 |
self.compile_group(files, handle_stats=handle_stats) |
| 683 |
for locale, stats in all_stats.items(): |
| 684 |
dump_json(stats, self.j(srcbase, locale, 'stats.json')) |
| 685 |
total = stats['translated'] + stats['untranslated'] |
| 686 |
# Raise the 30% threshold in the future |
| 687 |
if total and (stats['translated'] / float(total)) > 0.3: |
| 688 |
complete[locale] = stats |
| 689 |
dump_json(complete, self.j(destbase, 'completed.json')) |
| 690 |
|
| 691 |
def clean(self): |
| 692 |
if os.path.exists(self.stats): |
| 693 |
os.remove(self.stats) |
| 694 |
zf = self.DEST + '.zip' |
| 695 |
if os.path.exists(zf): |
| 696 |
os.remove(zf) |
| 697 |
destbase = self.j(self.d(self.SRC), 'manual', 'locale') |
| 698 |
if os.path.exists(destbase): |
| 699 |
shutil.rmtree(destbase) |
| 700 |
shutil.rmtree(self.cache_dir) |
| 701 |
|
| 702 |
|
| 703 |
# }}} |
| 704 |
|
| 705 |
|
| 706 |
class GetTranslations(Translations): # {{{ |
| 707 |
description = 'Get updated translations from Transifex' |
| 708 |
|
| 709 |
@property |
| 710 |
def is_modified(self): |
| 711 |
return bool(subprocess.check_output('git status --porcelain'.split(), cwd=self.TRANSLATIONS)) |
| 712 |
|
| 713 |
def add_options(self, parser): |
| 714 |
parser.add_option('-e', '--check-for-errors', default=False, action='store_true', help='Only check for errors in .po files') |
| 715 |
|
| 716 |
def run(self, opts): |
| 717 |
require_git_master() |
| 718 |
if opts.check_for_errors: |
| 719 |
self.check_all() |
| 720 |
return |
| 721 |
self.tx('pull -a') |
| 722 |
if not self.is_modified: |
| 723 |
self.info('No translations were updated') |
| 724 |
return |
| 725 |
self.upload_to_vcs() |
| 726 |
self.check_all() |
| 727 |
|
| 728 |
def check_all(self): |
| 729 |
self.check_for_errors() |
| 730 |
self.check_for_user_manual_errors() |
| 731 |
if self.is_modified: |
| 732 |
self.upload_to_vcs('Fixed translations') |
| 733 |
|
| 734 |
def check_for_user_manual_errors(self): |
| 735 |
sys.path.insert(0, self.j(self.d(self.SRC), 'setup')) |
| 736 |
import polib |
| 737 |
|
| 738 |
del sys.path[0] |
| 739 |
self.info('Checking user manual translations...') |
| 740 |
srcbase = self.j(self.d(self.SRC), 'translations', 'manual') |
| 741 |
changes = defaultdict(set) |
| 742 |
for lang in os.listdir(srcbase): |
| 743 |
if lang.startswith('en_') or lang == 'en': |
| 744 |
continue |
| 745 |
q = self.j(srcbase, lang) |
| 746 |
if not os.path.isdir(q): |
| 747 |
continue |
| 748 |
for po in os.listdir(q): |
| 749 |
if not po.endswith('.po'): |
| 750 |
continue |
| 751 |
f = polib.pofile(os.path.join(q, po)) |
| 752 |
changed = False |
| 753 |
for entry in f.translated_entries(): |
| 754 |
if '`generated/en/' in entry.msgstr: |
| 755 |
changed = True |
| 756 |
entry.msgstr = entry.msgstr.replace('`generated/en/', '`generated/' + lang + '/') |
| 757 |
bname = os.path.splitext(po)[0] |
| 758 |
slug = 'user_manual_' + bname |
| 759 |
changes[slug].add(lang) |
| 760 |
if changed: |
| 761 |
f.save() |
| 762 |
for slug, languages in changes.items(): |
| 763 |
print('Pushing fixes for languages: {} in {}'.format(', '.join(languages), slug)) |
| 764 |
self.tx('push -r calibre.{} -t -l {}'.format(slug, ','.join(languages))) |
| 765 |
|
| 766 |
def check_for_errors(self): |
| 767 |
self.info('Checking for errors in .po files...') |
| 768 |
groups = 'calibre content-server website'.split() |
| 769 |
for group in groups: |
| 770 |
self.check_group(group) |
| 771 |
self.check_website() |
| 772 |
for group in groups: |
| 773 |
self.push_fixes(group) |
| 774 |
|
| 775 |
def push_fixes(self, group): |
| 776 |
languages = set() |
| 777 |
for line in subprocess.check_output('git status --porcelain'.split(), cwd=self.TRANSLATIONS).decode('utf-8').splitlines(): |
| 778 |
parts = line.strip().split() |
| 779 |
if len(parts) > 1 and 'M' in parts[0] and parts[-1].startswith(group + '/') and parts[-1].endswith('.po'): |
| 780 |
languages.add(os.path.basename(parts[-1]).partition('.')[0]) |
| 781 |
if languages: |
| 782 |
pot = 'main' if group == 'calibre' else group.replace('-', '_') |
| 783 |
print('Pushing fixes for {}.pot languages: {}'.format(pot, ', '.join(languages))) |
| 784 |
self.tx(f'push -r calibre.{pot} -t -l ' + ','.join(languages)) |
| 785 |
|
| 786 |
def check_group(self, group): |
| 787 |
files = glob.glob(os.path.join(self.TRANSLATIONS, group, '*.po')) |
| 788 |
SKIP_BAD_LANGUAGES = ( |
| 789 |
( |
| 790 |
'ko', # too many errors in the translations |
| 791 |
) |
| 792 |
if group == 'website' |
| 793 |
else () |
| 794 |
) |
| 795 |
files = [x for x in files if os.path.basename(x).rpartition('.')[0] not in SKIP_BAD_LANGUAGES] |
| 796 |
cmd = ['msgfmt', '-o', os.devnull, '--check-format'] |
| 797 |
# Disabled because too many such errors, and not that critical anyway |
| 798 |
# if group == 'calibre': |
| 799 |
# cmd += ['--check-accelerators=&'] |
| 800 |
|
| 801 |
def check(f): |
| 802 |
p = subprocess.Popen(cmd + [f], stderr=subprocess.PIPE) |
| 803 |
errs = p.stderr.read() |
| 804 |
p.wait() |
| 805 |
return errs |
| 806 |
|
| 807 |
def check_for_control_chars(f): |
| 808 |
with open(f, 'rb') as f: |
| 809 |
raw = f.read().decode('utf-8') |
| 810 |
pat = re.compile(r'[\0-\x08\x0b\x0c\x0e-\x1f\x7f\x80-\x9f]') |
| 811 |
errs = [] |
| 812 |
for i, line in enumerate(raw.splitlines()): |
| 813 |
if pat.search(line) is not None: |
| 814 |
errs.append(f'There are ASCII control codes on line number: {i + 1}') |
| 815 |
return '\n'.join(errs) |
| 816 |
|
| 817 |
for f in files: |
| 818 |
errs = check(f) |
| 819 |
if errs: |
| 820 |
print(f) |
| 821 |
print(errs) |
| 822 |
edit_file(f) |
| 823 |
if check(f): |
| 824 |
raise SystemExit('Aborting as not all errors were fixed') |
| 825 |
errs = check_for_control_chars(f) |
| 826 |
if errs: |
| 827 |
print(f, 'has ASCII control codes in it') |
| 828 |
print(errs) |
| 829 |
raise SystemExit(1) |
| 830 |
|
| 831 |
def check_website(self): |
| 832 |
errors = os.path.join(tempfile.gettempdir(), 'calibre-translation-errors') |
| 833 |
if os.path.exists(errors): |
| 834 |
shutil.rmtree(errors) |
| 835 |
os.mkdir(errors) |
| 836 |
tpath = self.j(self.TRANSLATIONS, 'website') |
| 837 |
pofilter = ('pofilter', '-i', tpath, '-o', errors, '-t', 'xmltags') |
| 838 |
subprocess.check_call(pofilter) |
| 839 |
errfiles = glob.glob(errors + os.sep + '*.po') |
| 840 |
if errfiles: |
| 841 |
subprocess.check_call([os.environ.get('EDITOR', 'vim'), '-f', '-p', '--'] + errfiles) |
| 842 |
for f in errfiles: |
| 843 |
with open(f, 'r+b') as f: |
| 844 |
raw = f.read() |
| 845 |
raw = re.sub(rb'# \(pofilter\).*', b'', raw) |
| 846 |
f.seek(0) |
| 847 |
f.truncate() |
| 848 |
f.write(raw) |
| 849 |
|
| 850 |
subprocess.check_call(['pomerge', '-t', tpath, '-i', errors, '-o', tpath]) |
| 851 |
|
| 852 |
def upload_to_vcs(self, msg=None): |
| 853 |
self.info('Uploading updated translations to version control') |
| 854 |
cc = partial(subprocess.check_call, cwd=self.TRANSLATIONS) |
| 855 |
cc('git add */*.po'.split()) |
| 856 |
cc('git commit -am'.split() + [msg or 'Updated translations']) |
| 857 |
cc('git push'.split()) |
| 858 |
|
| 859 |
|
| 860 |
# }}} |
| 861 |
|
| 862 |
|
| 863 |
class ISO639(Command): # {{{ |
| 864 |
description = 'Compile language code maps for performance' |
| 865 |
sub_commands = ['iso_data'] |
| 866 |
DEST = os.path.join(os.path.dirname(POT.SRC), 'resources', 'localization', 'iso639.calibre_msgpack') |
| 867 |
|
| 868 |
def run(self, opts): |
| 869 |
dest = self.DEST |
| 870 |
base = self.d(dest) |
| 871 |
if not os.path.exists(base): |
| 872 |
os.makedirs(base) |
| 873 |
self.info('Packing ISO-639 codes to', dest) |
| 874 |
root = json.loads(iso_data.db_data('iso_639-3.json')) |
| 875 |
entries = root['639-3'] |
| 876 |
by_2 = {} |
| 877 |
by_3 = {} |
| 878 |
m2to3 = {} |
| 879 |
m3to2 = {} |
| 880 |
nm = {} |
| 881 |
codes2, codes3 = set(), set() |
| 882 |
for x in entries: |
| 883 |
two = x.get('alpha_2') |
| 884 |
if two: |
| 885 |
two = str(two) |
| 886 |
threeb = x.get('alpha_3') |
| 887 |
if threeb: |
| 888 |
threeb = str(threeb) |
| 889 |
if threeb is None: |
| 890 |
continue |
| 891 |
name = x.get('inverted_name') or x.get('name') |
| 892 |
if name: |
| 893 |
name = str(name) |
| 894 |
if not name or name[0] in '!~=/\'"': |
| 895 |
continue |
| 896 |
|
| 897 |
if two is not None: |
| 898 |
by_2[two] = name |
| 899 |
codes2.add(two) |
| 900 |
m2to3[two] = threeb |
| 901 |
m3to2[threeb] = two |
| 902 |
codes3.add(threeb) |
| 903 |
by_3[threeb] = name |
| 904 |
base_name = name.lower() |
| 905 |
nm[base_name] = threeb |
| 906 |
|
| 907 |
x = {'by_2': by_2, 'by_3': by_3, 'codes2': codes2, 'codes3': codes3, '2to3': m2to3, '3to2': m3to2, 'name_map': nm} |
| 908 |
from calibre.utils.serialize import msgpack_dumps |
| 909 |
|
| 910 |
with open(dest, 'wb') as f: |
| 911 |
f.write(msgpack_dumps(x)) |
| 912 |
|
| 913 |
def clean(self): |
| 914 |
if os.path.exists(self.DEST): |
| 915 |
os.remove(self.DEST) |
| 916 |
|
| 917 |
|
| 918 |
# }}} |
| 919 |
|
| 920 |
|
| 921 |
class ISO3166(ISO639): # {{{ |
| 922 |
description = 'Compile country code maps for performance' |
| 923 |
sub_commands = ['iso_data'] |
| 924 |
DEST = os.path.join(os.path.dirname(POT.SRC), 'resources', 'localization', 'iso3166.calibre_msgpack') |
| 925 |
|
| 926 |
def run(self, opts): |
| 927 |
dest = self.DEST |
| 928 |
base = self.d(dest) |
| 929 |
if not os.path.exists(base): |
| 930 |
os.makedirs(base) |
| 931 |
self.info('Packing ISO-3166 codes to', dest) |
| 932 |
db = json.loads(iso_data.db_data('iso_3166-1.json')) |
| 933 |
codes = set() |
| 934 |
three_map = {} |
| 935 |
name_map = {} |
| 936 |
for x in db['3166-1']: |
| 937 |
two = x.get('alpha_2') |
| 938 |
if two: |
| 939 |
two = str(two) |
| 940 |
codes.add(two) |
| 941 |
name_map[two] = x.get('common_name') or x.get('name') |
| 942 |
if name_map[two]: |
| 943 |
name_map[two] = str(name_map[two]) |
| 944 |
three = x.get('alpha_3') |
| 945 |
if three: |
| 946 |
three_map[str(three)] = two |
| 947 |
x = {'names': name_map, 'codes': frozenset(codes), 'three_map': three_map} |
| 948 |
from calibre.utils.serialize import msgpack_dumps |
| 949 |
|
| 950 |
with open(dest, 'wb') as f: |
| 951 |
f.write(msgpack_dumps(x)) |
| 952 |
|
| 953 |
|
| 954 |
# }}} |