Debian Codegraph @

setup/translations.py

raw
1 #!/usr/bin/env python
2
3
4 __license__ = 'GPL v3'
5 __copyright__ = '2009, Kovid Goyal <[email protected]>'
6 __docformat__ = 'restructuredtext en'
7
8 import errno
9 import glob
10 import hashlib
11 import inspect
12 import json
13 import os
14 import re
15 import shlex
16 import shutil
17 import subprocess
18 import sys
19 import tempfile
20 import textwrap
21 import time
22 from collections import defaultdict
23 from functools import lru_cache, partial
24 from locale import normalize as normalize_locale
25
26 from setup import Command, __appname__, __version__, build_cache_dir, dump_json, edit_file, is_ci, require_git_master
27 from setup.iso_codes import iso_data
28 from setup.parallel_build import batched_parallel_jobs
29
30
31 def serialize_msgid(text):
32 """Serialize a string in the format used by msgid in GNU POT files."""
33 if not text:
34 return 'msgid ""\n'
35 # Escape backslashes and quotes
36 escaped = text.replace('\\', r'\\').replace('"', r'\"')
37 ans = ['msgid ""']
38 lines = escaped.splitlines()
39 for line in lines:
40 trailer = '"' if line is lines[-1] else r'\n"'
41 ans.append(f'"{line}{trailer}')
42 return '\n'.join(ans)
43
44
45 def qt_sources():
46 qtdir = os.environ.get('QT_SRC', '/usr/src/qt6/qtbase')
47 j = partial(os.path.join, qtdir)
48 return list(
49 map(
50 j,
51 [
52 'src/gui/kernel/qplatformtheme.cpp',
53 'src/widgets/dialogs/qcolordialog.cpp',
54 'src/widgets/dialogs/qfontdialog.cpp',
55 'src/widgets/widgets/qscrollbar.cpp',
56 ],
57 )
58 )
59
60
61 @lru_cache(maxsize=2)
62 def tx_exe():
63 return os.environ.get('TX', shutil.which('tx-cli') or shutil.which('tx') or 'tx')
64
65
66 class POT(Command): # {{{
67 description = 'Update the .pot translation template and upload it'
68 TRANSLATIONS = os.path.join(os.path.dirname(Command.SRC), 'translations')
69 MANUAL = os.path.join(os.path.dirname(Command.SRC), 'manual')
70
71 def tx(self, cmd, **kw):
72 kw['cwd'] = kw.get('cwd', self.TRANSLATIONS)
73 if hasattr(cmd, 'format'):
74 cmd = shlex.split(cmd)
75 cmd = [tx_exe()] + cmd
76 self.info(' '.join(cmd))
77 return subprocess.check_call(cmd, **kw)
78
79 def git(self, cmd, **kw):
80 kw['cwd'] = kw.get('cwd', self.TRANSLATIONS)
81 if hasattr(cmd, 'format'):
82 cmd = shlex.split(cmd)
83 f = getattr(subprocess, ('call' if kw.pop('use_call', False) else 'check_call'))
84 return f(['git'] + cmd, **kw)
85
86 def upload_pot(self, resource):
87 self.tx(['push', '-r', 'calibre.' + resource, '-s'], cwd=self.TRANSLATIONS)
88
89 def source_files(self):
90 ans = [self.a(self.j(self.MANUAL, x)) for x in ('custom.py', 'conf.py')]
91 for root, _, files in os.walk(self.j(self.SRC, __appname__)):
92 for name in files:
93 if name.endswith('.py'):
94 ans.append(self.a(self.j(root, name)))
95 return ans
96
97 def get_ffml_docs(self):
98 from calibre.gui2.dialogs.template_general_info import ffml_doc, general_doc
99 from calibre.utils.formatter_functions import formatter_functions
100
101 all_funcs = formatter_functions().get_builtins()
102 ans = []
103 for ff in all_funcs.values():
104 lnum = inspect.getsourcelines(ff.__doc__getter__)[1]
105 text = ff.__doc__getter__().msgid
106 ans.append(f'#: src/calibre/utils/formatter_function.py:{lnum}\n' + serialize_msgid(text) + '\nmsgstr ""\n\n')
107 for ff in (ffml_doc, general_doc):
108 lnum = inspect.getsourcelines(ff)[1]
109 text = ff().msgid
110 ans.append(f'#: src/calibre/gui2/dialogs/template_general_info.py:{lnum}\n' + serialize_msgid(text) + '\nmsgstr ""\n\n')
111 return ''.join(ans)
112
113 def get_tweaks_docs(self):
114 path = self.a(self.j(self.SRC, '..', 'resources', 'default_tweaks.py'))
115 with open(path, 'rb') as f:
116 raw = f.read().decode('utf-8')
117 msgs = []
118 lines = list(raw.splitlines())
119 for i, line in enumerate(lines):
120 if line.startswith('#:'):
121 msgs.append((i, line[2:].strip()))
122 j = i
123 block = []
124 while True:
125 j += 1
126 line = lines[j]
127 if not line.startswith('#'):
128 break
129 block.append(line[1:].strip())
130 if block:
131 msgs.append((i + 1, '\n'.join(block)))
132
133 ans = []
134 for lineno, msg in msgs:
135 ans.append(f'#: {path}:{lineno}')
136 slash = chr(92)
137 msg = msg.replace(slash, slash * 2).replace('"', r'\"').replace('\n', r'\n').replace('\r', r'\r').replace('\t', r'\t')
138 ans.append(f'msgid "{msg}"')
139 ans.append('msgstr ""')
140 ans.append('')
141
142 return '\n'.join(ans)
143
144 def get_content_server_strings(self):
145 self.info('Generating translation template for content_server')
146 from calibre import walk
147 from calibre.utils.rapydscript import create_pot
148
149 files = (f for f in walk(self.j(self.SRC, 'pyj')) if f.endswith('.pyj'))
150 pottext = create_pot(files).encode('utf-8')
151 dest = self.j(self.TRANSLATIONS, 'content-server', 'content-server.pot')
152 with open(dest, 'wb') as f:
153 f.write(pottext)
154 self.upload_pot(resource='content_server')
155 self.git(['add', dest])
156
157 def get_user_manual_docs(self):
158 self.info('Generating translation templates for user_manual')
159 base = tempfile.mkdtemp()
160 subprocess.check_call([sys.executable, self.j(self.d(self.SRC), 'manual', 'build.py'), 'gettext', base])
161 tbase = self.j(self.TRANSLATIONS, 'manual')
162 for x in os.listdir(base):
163 if not x.endswith('.pot'):
164 continue
165 src, dest = self.j(base, x), self.j(tbase, x)
166 needs_import = not os.path.exists(dest)
167 with open(src, 'rb') as s, open(dest, 'wb') as d:
168 shutil.copyfileobj(s, d)
169 bname = os.path.splitext(x)[0]
170 slug = 'user_manual_' + bname
171 if needs_import:
172 self.tx(['set', '-r', 'calibre.' + slug, '--source', '-l', 'en', '-t', 'PO', dest])
173 with open(self.j(self.d(tbase), '.tx/config'), 'r+b') as f:
174 lines = f.read().decode('utf-8').splitlines()
175 for i in range(len(lines)):
176 line = lines[i].strip()
177 if line == f'[calibre.{slug}]':
178 lines.insert(i + 1, f'file_filter = manual/<lang>/{bname}.po')
179 f.seek(0), f.truncate(), f.write('\n'.join(lines).encode('utf-8'))
180 break
181 else:
182 raise SystemExit(f'Failed to add file_filter for slug={slug} to config file')
183 self.git('add .tx/config')
184 self.upload_pot(resource=slug)
185 self.git(['add', dest])
186 shutil.rmtree(base)
187
188 def get_website_strings(self):
189 self.info('Generating translation template for website')
190 self.wn_path = os.path.expanduser('~/work/srv/main/static/generate.py')
191 data = subprocess.check_output([self.wn_path, '--pot', '/tmp/wn'])
192 data = json.loads(data)
193
194 def do(name):
195 messages = data[name]
196 bdir = os.path.join(self.TRANSLATIONS, name)
197 if not os.path.exists(bdir):
198 os.makedirs(bdir)
199 pot = os.path.abspath(os.path.join(bdir, name + '.pot'))
200 with open(pot, 'wb') as f:
201 f.write(self.pot_header().encode('utf-8'))
202 f.write(b'\n')
203 f.write('\n'.join(messages).encode('utf-8'))
204 self.upload_pot(resource=name)
205 self.git(['add', pot])
206
207 do('website')
208 do('changelog')
209
210 def pot_header(self, appname=__appname__, version=__version__):
211 return textwrap.dedent('''\
212 # Translation template file..
213 # Copyright (C) {year} Kovid Goyal
214 # Kovid Goyal <[email protected]>, {year}.
215 #
216 msgid ""
217 msgstr ""
218 "Project-Id-Version: {appname} {version}\\n"
219 "POT-Creation-Date: {time}\\n"
220 "PO-Revision-Date: {time}\\n"
221 "Last-Translator: Automatically generated\\n"
222 "Language-Team: LANGUAGE\\n"
223 "MIME-Version: 1.0\\n"
224 "Report-Msgid-Bugs-To: https://bugs.launchpad.net/calibre\\n"
225 "Plural-Forms: nplurals=INTEGER; plural=EXPRESSION;\\n"
226 "Content-Type: text/plain; charset=UTF-8\\n"
227 "Content-Transfer-Encoding: 8bit\\n"
228
229 ''').format(appname=appname, version=version, year=time.strftime('%Y'), time=time.strftime('%Y-%m-%d %H:%M+%Z'))
230
231 def run(self, opts):
232 if not is_ci:
233 require_git_master()
234 if not is_ci:
235 self.get_website_strings()
236 self.get_content_server_strings()
237 self.get_user_manual_docs()
238 files = self.source_files()
239 qt_inputs = qt_sources()
240 pot_header = self.pot_header()
241
242 with tempfile.NamedTemporaryFile() as fl:
243 fl.write('\n'.join(files).encode('utf-8'))
244 fl.flush()
245 out = tempfile.NamedTemporaryFile(suffix='.pot', delete=False)
246 out.close()
247 self.info('Creating translations template...')
248 subprocess.check_call([
249 'xgettext',
250 '-f',
251 fl.name,
252 '--default-domain=calibre',
253 '-o',
254 out.name,
255 '-L',
256 'Python',
257 '--from-code=UTF-8',
258 '--sort-by-file',
259 '--omit-header',
260 '--no-wrap',
261 '-k__',
262 '-kpgettext:1c,2',
263 '--add-comments=NOTE:',
264 ])
265 subprocess.check_call(
266 [
267 'xgettext',
268 '-j',
269 '--default-domain=calibre',
270 '-o',
271 out.name,
272 '--from-code=UTF-8',
273 '--sort-by-file',
274 '--omit-header',
275 '--no-wrap',
276 '-kQT_TRANSLATE_NOOP:2',
277 '-ktr',
278 '-ktranslate:2',
279 ]
280 + qt_inputs
281 )
282 with open(out.name, 'rb') as f:
283 src = f.read().decode('utf-8')
284 os.remove(out.name)
285 src = pot_header + '\n' + src
286 src += '\n\n' + self.get_tweaks_docs()
287 src += '\n\n' + self.get_ffml_docs()
288 bdir = os.path.join(self.TRANSLATIONS, __appname__)
289 if not os.path.exists(bdir):
290 os.makedirs(bdir)
291 pot = os.path.join(bdir, 'main.pot')
292 # Workaround for bug in xgettext:
293 # https://savannah.gnu.org/bugs/index.php?41668
294 src = re.sub(r'#, python-brace-format\s+msgid ""\s+.*<code>{0:</code>', lambda m: m.group().replace('python-brace', 'no-python-brace'), src)
295 with open(pot, 'wb') as f:
296 f.write(src.encode('utf-8'))
297 self.info('Translations template:', os.path.abspath(pot))
298 self.upload_pot(resource='main')
299 self.git(['add', os.path.abspath(pot)])
300
301 if not is_ci and self.git('diff-index --cached --quiet --ignore-submodules HEAD --', use_call=True) != 0:
302 self.git(['commit', '-m', 'Updated translation templates'])
303 self.git('push')
304
305 return pot
306
307
308 # }}}
309
310
311 class Translations(POT): # {{{
312 description = '''Compile the translations'''
313 DEST = os.path.join(os.path.dirname(POT.SRC), 'resources', 'localization', 'locales')
314
315 @property
316 def cache_dir(self):
317 ans = self.j(build_cache_dir(), 'translations')
318 if not hasattr(self, 'cache_dir_created'):
319 self.cache_dir_created = True
320 try:
321 os.mkdir(ans)
322 except OSError as err:
323 if err.errno != errno.EEXIST:
324 raise
325 return ans
326
327 def cache_name(self, f):
328 f = os.path.relpath(f, self.d(self.SRC))
329 return f.replace(os.sep, '.').replace('/', '.').lstrip('.')
330
331 def read_cache(self, f):
332 cname = self.cache_name(f)
333 try:
334 with open(self.j(self.cache_dir, cname), 'rb') as f:
335 data = f.read()
336 return data[:20], data[20:]
337 except OSError as err:
338 if err.errno != errno.ENOENT:
339 raise
340 return None, None
341
342 def write_cache(self, data, h, f):
343 cname = self.cache_name(f)
344 assert len(h) == 20
345 with open(self.j(self.cache_dir, cname), 'wb') as f:
346 f.write(h), f.write(data)
347
348 def is_po_file_ok(self, x):
349 bname = os.path.splitext(os.path.basename(x))[0]
350 # [email protected] is identical to sr.po. And we don't support country
351 # specific variants except for a few.
352 if '_' in bname:
353 return bname.partition('_')[0] in ('pt', 'zh', 'bn')
354 return bname != 'sr@latin'
355
356 def po_files(self):
357 return [x for x in glob.glob(os.path.join(self.TRANSLATIONS, __appname__, '*.po')) if self.is_po_file_ok(x)]
358
359 def mo_file(self, po_file):
360 locale = os.path.splitext(os.path.basename(po_file))[0]
361 return locale, os.path.join(self.DEST, locale, 'messages.mo')
362
363 def run(self, opts):
364 self.compile_main_translations()
365 self.compile_content_server_translations()
366 self.freeze_locales()
367 self.compile_user_manual_translations()
368 self.compile_website_translations()
369 self.compile_changelog_translations()
370
371 def compile_group(self, files, handle_stats=None, action_per_file=None, make_translated_strings_unique=False, keyfunc=lambda x: x):
372 ok_files = []
373 hashmap = {}
374
375 def stats_cache(src, data=None):
376 cname = self.cache_name(keyfunc(src)) + '.stats.json'
377 with open(self.j(self.cache_dir, cname), ('rb' if data is None else 'wb')) as f:
378 if data is None:
379 return json.loads(f.read())
380 data = json.dumps(data)
381 if not isinstance(data, bytes):
382 data = data.encode('utf-8')
383 f.write(data)
384
385 for src, dest in files:
386 base = os.path.dirname(dest)
387 if not os.path.exists(base):
388 os.makedirs(base)
389 data, h = self.hash_and_data(src, keyfunc)
390 current_hash = h.digest()
391 saved_hash, saved_data = self.read_cache(keyfunc(src))
392 if current_hash == saved_hash:
393 with open(dest, 'wb') as d:
394 d.write(saved_data)
395 if handle_stats is not None:
396 handle_stats(src, stats_cache(src))
397 else:
398 ok_files.append((src, dest))
399 hashmap[keyfunc(src)] = current_hash
400 if action_per_file is not None:
401 action_per_file(src)
402
403 self.info(f'\tCompiling {len(ok_files)} files')
404 items = []
405 results = batched_parallel_jobs(
406 [sys.executable, self.j(self.SRC, 'calibre', 'translations', 'msgfmt.py'), 'STDIN', 'uniqify' if make_translated_strings_unique else ' '], ok_files
407 )
408 for (src, dest), data in zip(ok_files, results):
409 items.append((src, dest, data))
410
411 for src, dest, data in items:
412 self.write_cache(open(dest, 'rb').read(), hashmap[keyfunc(src)], keyfunc(src))
413 stats_cache(src, data)
414 if handle_stats is not None:
415 handle_stats(src, data)
416
417 def compile_main_translations(self):
418 l = {}
419 lc_dataf = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'lc_data.py')
420 exec(compile(open(lc_dataf, 'rb').read(), lc_dataf, 'exec'), l, l)
421 lcdata = {k: dict(v) for k, v in l['data']}
422 self.info('Compiling main UI translation files...')
423 fmap = {f: self.mo_file(f) for f in self.po_files()}
424 files = [(f, fmap[f][1]) for f in self.po_files()]
425
426 def action_per_file(f):
427 locale, dest = fmap[f]
428 ln = normalize_locale(locale).partition('.')[0]
429 if ln in lcdata:
430 ld = lcdata[ln]
431 lcdest = self.j(self.d(dest), 'lcdata.calibre_msgpack')
432 from calibre.utils.serialize import msgpack_dumps
433
434 with open(lcdest, 'wb') as lcf:
435 lcf.write(msgpack_dumps(ld))
436
437 stats = {}
438
439 def handle_stats(f, data):
440 trans, untrans = data['translated'], data['untranslated']
441 total = trans + untrans
442 locale = fmap[f][0]
443 stats[locale] = min(1.0, float(trans) / total)
444
445 self.compile_group(files, handle_stats=handle_stats, action_per_file=action_per_file)
446
447 self.info('Compiling ISO639 files...')
448 files = []
449 skip_iso = {
450 'si',
451 'te',
452 'km',
453 'en_GB',
454 'en_AU',
455 'en_CA',
456 'yi',
457 'ku',
458 'my',
459 'uz@Latn',
460 'fil',
461 'hy',
462 'ltg',
463 'km_KH',
464 'km',
465 'ur',
466 'ml',
467 'fo',
468 'ug',
469 'jv',
470 'nds',
471 }
472
473 def handle_stats(f, data):
474 if False and data['uniqified']:
475 print(f'{data["uniqified"]:3d} non-unique language name translations in {os.path.basename(f)}', file=sys.stderr)
476
477 with tempfile.TemporaryDirectory() as tdir:
478 iso_data.extract_po_files('iso_639-3', tdir)
479 for f, (locale, dest) in fmap.items():
480 iscpo = {'zh_HK': 'zh_CN'}.get(locale, locale)
481 iso639 = self.j(tdir, f'{iscpo}.po')
482 if os.path.exists(iso639):
483 files.append((iso639, self.j(self.d(dest), 'iso639.mo')))
484 else:
485 iscpo = iscpo.partition('_')[0]
486 iso639 = self.j(tdir, f'{iscpo}.po')
487 if os.path.exists(iso639):
488 files.append((iso639, self.j(self.d(dest), 'iso639.mo')))
489 elif locale not in skip_iso:
490 self.warn('No ISO 639 translations for locale:', locale)
491 self.compile_group(
492 files,
493 make_translated_strings_unique=True,
494 handle_stats=handle_stats,
495 keyfunc=lambda x: os.path.join(self.d(self.SRC), 'iso639', os.path.basename(x)),
496 )
497
498 self.info('Compiling ISO3166 files...')
499 files = []
500 skip_iso = {'en_GB', 'en_AU', 'en_CA', 'yi', 'ku', 'uz@Latn', 'ltg', 'nds', 'jv'}
501 with tempfile.TemporaryDirectory() as tdir:
502 iso_data.extract_po_files('iso_3166-1', tdir)
503 for f, (locale, dest) in fmap.items():
504 pofile = self.j(tdir, f'{locale}.po')
505 if os.path.exists(pofile):
506 files.append((pofile, self.j(self.d(dest), 'iso3166.mo')))
507 else:
508 pofile = self.j(tdir, f'{locale.partition("_")[0]}.po')
509 if os.path.exists(pofile):
510 files.append((pofile, self.j(self.d(dest), 'iso3166.mo')))
511 elif locale not in skip_iso:
512 self.warn('No ISO 3166 translations for locale:', locale)
513 self.compile_group(
514 files,
515 make_translated_strings_unique=True,
516 handle_stats=lambda f, d: None,
517 keyfunc=lambda x: os.path.join(self.d(self.SRC), 'iso3166', os.path.basename(x)),
518 )
519
520 dest = self.stats
521 base = self.d(dest)
522 try:
523 os.mkdir(base)
524 except OSError as err:
525 if err.errno != errno.EEXIST:
526 raise
527 from calibre.utils.serialize import msgpack_dumps
528
529 with open(dest, 'wb') as f:
530 f.write(msgpack_dumps(stats))
531
532 def hash_and_data(self, f, keyfunc=lambda x: x):
533 with open(f, 'rb') as s:
534 data = s.read()
535 h = hashlib.sha1(data)
536 h.update(keyfunc(f).encode('utf-8'))
537 return data, h
538
539 def compile_content_server_translations(self):
540 self.info('Compiling content-server translations')
541 from calibre.utils.rapydscript import msgfmt
542 from calibre.utils.zipfile import ZIP_DEFLATED, ZIP_STORED, ZipFile, ZipInfo
543
544 with ZipFile(self.j(self.RESOURCES, 'content-server', 'locales.zip'), 'w', ZIP_DEFLATED) as zf:
545 for src in glob.glob(os.path.join(self.TRANSLATIONS, 'content-server', '*.po')):
546 if not self.is_po_file_ok(src):
547 continue
548 data, h = self.hash_and_data(src)
549 current_hash = h.digest()
550 saved_hash, saved_data = self.read_cache(src)
551 if current_hash == saved_hash:
552 raw = saved_data
553 else:
554 # self.info('\tParsing ' + os.path.basename(src))
555 raw = None
556 po_data = data.decode('utf-8')
557 data = json.loads(msgfmt(po_data))
558 translated_entries = {k: v for k, v in data['entries'].items() if v and sum(map(len, v))}
559 data['entries'] = translated_entries
560 data['hash'] = h.hexdigest()
561 cdata = b'{}'
562 if translated_entries:
563 raw = json.dumps(data, ensure_ascii=False, sort_keys=True)
564 if isinstance(raw, str):
565 raw = raw.encode('utf-8')
566 cdata = raw
567 self.write_cache(cdata, current_hash, src)
568 if raw:
569 zi = ZipInfo(os.path.basename(src).rpartition('.')[0])
570 zi.compress_type = ZIP_STORED if is_ci else ZIP_DEFLATED
571 zf.writestr(zi, raw)
572
573 def freeze_locales(self):
574 zf = self.DEST + '.zip'
575 from calibre import CurrentDir
576 from calibre.utils.zipfile import ZIP_DEFLATED, ZipFile
577
578 with ZipFile(zf, 'w', ZIP_DEFLATED) as zf:
579 with CurrentDir(self.DEST):
580 zf.add_dir('.')
581 shutil.rmtree(self.DEST)
582
583 @property
584 def stats(self):
585 return self.j(self.d(self.DEST), 'stats.calibre_msgpack')
586
587 def _compile_website_translations(self, name='website', threshold=50):
588 from calibre.ptempfile import TemporaryDirectory
589 from calibre.utils.localization import get_language, translator_for_lang
590 from calibre.utils.zipfile import ZIP_STORED, ZipFile, ZipInfo
591
592 self.info('Compiling', name, 'translations...')
593 srcbase = self.j(self.d(self.SRC), 'translations', name)
594 if not os.path.exists(srcbase):
595 os.makedirs(srcbase)
596 fmap = {}
597 files = []
598 stats = {}
599 done = []
600
601 def handle_stats(src, data):
602 locale = fmap[src]
603 trans, untrans = data['translated'], data['untranslated']
604 total = trans + untrans
605 stats[locale] = round(100 * trans / total)
606
607 with TemporaryDirectory() as tdir, ZipFile(self.j(srcbase, 'locales.zip'), 'w', ZIP_STORED) as zf:
608 for f in os.listdir(srcbase):
609 if f.endswith('.po'):
610 if not self.is_po_file_ok(f):
611 continue
612 l = f.partition('.')[0]
613 pf = l.split('_')[0]
614 if pf == 'en':
615 continue
616 d = os.path.join(tdir, l + '.mo')
617 f = os.path.join(srcbase, f)
618 fmap[f] = l
619 files.append((f, d))
620 self.compile_group(files, handle_stats=handle_stats)
621
622 for locale, translated in stats.items():
623 if translated >= threshold:
624 with open(os.path.join(tdir, locale + '.mo'), 'rb') as f:
625 raw = f.read()
626 zi = ZipInfo(os.path.basename(f.name))
627 zi.compress_type = ZIP_STORED
628 zf.writestr(zi, raw)
629 done.append(locale)
630 dl = done + ['en']
631
632 lang_names = {}
633 for l in dl:
634 translator = translator_for_lang(l)['translator']
635 t = translator.gettext
636 t = partial(get_language, gettext_func=t)
637 lang_names[l] = {x: t(x) for x in dl}
638 zi = ZipInfo('lang-names.json')
639 zi.compress_type = ZIP_STORED
640 zf.writestr(zi, json.dumps(lang_names, ensure_ascii=False).encode('utf-8'))
641 return done
642
643 def compile_website_translations(self):
644 done = self._compile_website_translations()
645 dest = self.j(self.d(self.stats), 'website-languages.txt')
646 data = ' '.join(sorted(done))
647 if not isinstance(data, bytes):
648 data = data.encode('utf-8')
649 with open(dest, 'wb') as f:
650 f.write(data)
651
652 def compile_changelog_translations(self):
653 self._compile_website_translations('changelog', threshold=0)
654
655 def compile_user_manual_translations(self):
656 self.info('Compiling user manual translations...')
657 srcbase = self.j(self.d(self.SRC), 'translations', 'manual')
658 destbase = self.j(self.d(self.SRC), 'manual', 'locale')
659 complete = {}
660 all_stats = defaultdict(lambda: {'translated': 0, 'untranslated': 0})
661 files = []
662 for x in os.listdir(srcbase):
663 q = self.j(srcbase, x)
664 if not os.path.isdir(q) or not self.is_po_file_ok(q):
665 continue
666 dest = self.j(destbase, x, 'LC_MESSAGES')
667 if os.path.exists(dest):
668 shutil.rmtree(dest)
669 os.makedirs(dest)
670 for po in os.listdir(q):
671 if not po.endswith('.po'):
672 continue
673 mofile = self.j(dest, po.rpartition('.')[0] + '.mo')
674 files.append((self.j(q, po), mofile))
675
676 def handle_stats(src, data):
677 locale = self.b(self.d(src))
678 stats = all_stats[locale]
679 stats['translated'] += data['translated']
680 stats['untranslated'] += data['untranslated']
681
682 self.compile_group(files, handle_stats=handle_stats)
683 for locale, stats in all_stats.items():
684 dump_json(stats, self.j(srcbase, locale, 'stats.json'))
685 total = stats['translated'] + stats['untranslated']
686 # Raise the 30% threshold in the future
687 if total and (stats['translated'] / float(total)) > 0.3:
688 complete[locale] = stats
689 dump_json(complete, self.j(destbase, 'completed.json'))
690
691 def clean(self):
692 if os.path.exists(self.stats):
693 os.remove(self.stats)
694 zf = self.DEST + '.zip'
695 if os.path.exists(zf):
696 os.remove(zf)
697 destbase = self.j(self.d(self.SRC), 'manual', 'locale')
698 if os.path.exists(destbase):
699 shutil.rmtree(destbase)
700 shutil.rmtree(self.cache_dir)
701
702
703 # }}}
704
705
706 class GetTranslations(Translations): # {{{
707 description = 'Get updated translations from Transifex'
708
709 @property
710 def is_modified(self):
711 return bool(subprocess.check_output('git status --porcelain'.split(), cwd=self.TRANSLATIONS))
712
713 def add_options(self, parser):
714 parser.add_option('-e', '--check-for-errors', default=False, action='store_true', help='Only check for errors in .po files')
715
716 def run(self, opts):
717 require_git_master()
718 if opts.check_for_errors:
719 self.check_all()
720 return
721 self.tx('pull -a')
722 if not self.is_modified:
723 self.info('No translations were updated')
724 return
725 self.upload_to_vcs()
726 self.check_all()
727
728 def check_all(self):
729 self.check_for_errors()
730 self.check_for_user_manual_errors()
731 if self.is_modified:
732 self.upload_to_vcs('Fixed translations')
733
734 def check_for_user_manual_errors(self):
735 sys.path.insert(0, self.j(self.d(self.SRC), 'setup'))
736 import polib
737
738 del sys.path[0]
739 self.info('Checking user manual translations...')
740 srcbase = self.j(self.d(self.SRC), 'translations', 'manual')
741 changes = defaultdict(set)
742 for lang in os.listdir(srcbase):
743 if lang.startswith('en_') or lang == 'en':
744 continue
745 q = self.j(srcbase, lang)
746 if not os.path.isdir(q):
747 continue
748 for po in os.listdir(q):
749 if not po.endswith('.po'):
750 continue
751 f = polib.pofile(os.path.join(q, po))
752 changed = False
753 for entry in f.translated_entries():
754 if '`generated/en/' in entry.msgstr:
755 changed = True
756 entry.msgstr = entry.msgstr.replace('`generated/en/', '`generated/' + lang + '/')
757 bname = os.path.splitext(po)[0]
758 slug = 'user_manual_' + bname
759 changes[slug].add(lang)
760 if changed:
761 f.save()
762 for slug, languages in changes.items():
763 print('Pushing fixes for languages: {} in {}'.format(', '.join(languages), slug))
764 self.tx('push -r calibre.{} -t -l {}'.format(slug, ','.join(languages)))
765
766 def check_for_errors(self):
767 self.info('Checking for errors in .po files...')
768 groups = 'calibre content-server website'.split()
769 for group in groups:
770 self.check_group(group)
771 self.check_website()
772 for group in groups:
773 self.push_fixes(group)
774
775 def push_fixes(self, group):
776 languages = set()
777 for line in subprocess.check_output('git status --porcelain'.split(), cwd=self.TRANSLATIONS).decode('utf-8').splitlines():
778 parts = line.strip().split()
779 if len(parts) > 1 and 'M' in parts[0] and parts[-1].startswith(group + '/') and parts[-1].endswith('.po'):
780 languages.add(os.path.basename(parts[-1]).partition('.')[0])
781 if languages:
782 pot = 'main' if group == 'calibre' else group.replace('-', '_')
783 print('Pushing fixes for {}.pot languages: {}'.format(pot, ', '.join(languages)))
784 self.tx(f'push -r calibre.{pot} -t -l ' + ','.join(languages))
785
786 def check_group(self, group):
787 files = glob.glob(os.path.join(self.TRANSLATIONS, group, '*.po'))
788 SKIP_BAD_LANGUAGES = (
789 (
790 'ko', # too many errors in the translations
791 )
792 if group == 'website'
793 else ()
794 )
795 files = [x for x in files if os.path.basename(x).rpartition('.')[0] not in SKIP_BAD_LANGUAGES]
796 cmd = ['msgfmt', '-o', os.devnull, '--check-format']
797 # Disabled because too many such errors, and not that critical anyway
798 # if group == 'calibre':
799 # cmd += ['--check-accelerators=&']
800
801 def check(f):
802 p = subprocess.Popen(cmd + [f], stderr=subprocess.PIPE)
803 errs = p.stderr.read()
804 p.wait()
805 return errs
806
807 def check_for_control_chars(f):
808 with open(f, 'rb') as f:
809 raw = f.read().decode('utf-8')
810 pat = re.compile(r'[\0-\x08\x0b\x0c\x0e-\x1f\x7f\x80-\x9f]')
811 errs = []
812 for i, line in enumerate(raw.splitlines()):
813 if pat.search(line) is not None:
814 errs.append(f'There are ASCII control codes on line number: {i + 1}')
815 return '\n'.join(errs)
816
817 for f in files:
818 errs = check(f)
819 if errs:
820 print(f)
821 print(errs)
822 edit_file(f)
823 if check(f):
824 raise SystemExit('Aborting as not all errors were fixed')
825 errs = check_for_control_chars(f)
826 if errs:
827 print(f, 'has ASCII control codes in it')
828 print(errs)
829 raise SystemExit(1)
830
831 def check_website(self):
832 errors = os.path.join(tempfile.gettempdir(), 'calibre-translation-errors')
833 if os.path.exists(errors):
834 shutil.rmtree(errors)
835 os.mkdir(errors)
836 tpath = self.j(self.TRANSLATIONS, 'website')
837 pofilter = ('pofilter', '-i', tpath, '-o', errors, '-t', 'xmltags')
838 subprocess.check_call(pofilter)
839 errfiles = glob.glob(errors + os.sep + '*.po')
840 if errfiles:
841 subprocess.check_call([os.environ.get('EDITOR', 'vim'), '-f', '-p', '--'] + errfiles)
842 for f in errfiles:
843 with open(f, 'r+b') as f:
844 raw = f.read()
845 raw = re.sub(rb'# \(pofilter\).*', b'', raw)
846 f.seek(0)
847 f.truncate()
848 f.write(raw)
849
850 subprocess.check_call(['pomerge', '-t', tpath, '-i', errors, '-o', tpath])
851
852 def upload_to_vcs(self, msg=None):
853 self.info('Uploading updated translations to version control')
854 cc = partial(subprocess.check_call, cwd=self.TRANSLATIONS)
855 cc('git add */*.po'.split())
856 cc('git commit -am'.split() + [msg or 'Updated translations'])
857 cc('git push'.split())
858
859
860 # }}}
861
862
863 class ISO639(Command): # {{{
864 description = 'Compile language code maps for performance'
865 sub_commands = ['iso_data']
866 DEST = os.path.join(os.path.dirname(POT.SRC), 'resources', 'localization', 'iso639.calibre_msgpack')
867
868 def run(self, opts):
869 dest = self.DEST
870 base = self.d(dest)
871 if not os.path.exists(base):
872 os.makedirs(base)
873 self.info('Packing ISO-639 codes to', dest)
874 root = json.loads(iso_data.db_data('iso_639-3.json'))
875 entries = root['639-3']
876 by_2 = {}
877 by_3 = {}
878 m2to3 = {}
879 m3to2 = {}
880 nm = {}
881 codes2, codes3 = set(), set()
882 for x in entries:
883 two = x.get('alpha_2')
884 if two:
885 two = str(two)
886 threeb = x.get('alpha_3')
887 if threeb:
888 threeb = str(threeb)
889 if threeb is None:
890 continue
891 name = x.get('inverted_name') or x.get('name')
892 if name:
893 name = str(name)
894 if not name or name[0] in '!~=/\'"':
895 continue
896
897 if two is not None:
898 by_2[two] = name
899 codes2.add(two)
900 m2to3[two] = threeb
901 m3to2[threeb] = two
902 codes3.add(threeb)
903 by_3[threeb] = name
904 base_name = name.lower()
905 nm[base_name] = threeb
906
907 x = {'by_2': by_2, 'by_3': by_3, 'codes2': codes2, 'codes3': codes3, '2to3': m2to3, '3to2': m3to2, 'name_map': nm}
908 from calibre.utils.serialize import msgpack_dumps
909
910 with open(dest, 'wb') as f:
911 f.write(msgpack_dumps(x))
912
913 def clean(self):
914 if os.path.exists(self.DEST):
915 os.remove(self.DEST)
916
917
918 # }}}
919
920
921 class ISO3166(ISO639): # {{{
922 description = 'Compile country code maps for performance'
923 sub_commands = ['iso_data']
924 DEST = os.path.join(os.path.dirname(POT.SRC), 'resources', 'localization', 'iso3166.calibre_msgpack')
925
926 def run(self, opts):
927 dest = self.DEST
928 base = self.d(dest)
929 if not os.path.exists(base):
930 os.makedirs(base)
931 self.info('Packing ISO-3166 codes to', dest)
932 db = json.loads(iso_data.db_data('iso_3166-1.json'))
933 codes = set()
934 three_map = {}
935 name_map = {}
936 for x in db['3166-1']:
937 two = x.get('alpha_2')
938 if two:
939 two = str(two)
940 codes.add(two)
941 name_map[two] = x.get('common_name') or x.get('name')
942 if name_map[two]:
943 name_map[two] = str(name_map[two])
944 three = x.get('alpha_3')
945 if three:
946 three_map[str(three)] = two
947 x = {'names': name_map, 'codes': frozenset(codes), 'three_map': three_map}
948 from calibre.utils.serialize import msgpack_dumps
949
950 with open(dest, 'wb') as f:
951 f.write(msgpack_dumps(x))
952
953
954 # }}}