| 1 |
#! /usr/bin/env python3 |
| 2 |
|
| 3 |
"""pygettext -- Python equivalent of xgettext(1) |
| 4 |
|
| 5 |
Many systems (Solaris, Linux, Gnu) provide extensive tools that ease the |
| 6 |
internationalization of C programs. Most of these tools are independent of |
| 7 |
the programming language and can be used from within Python programs. |
| 8 |
Martin von Loewis' work[1] helps considerably in this regard. |
| 9 |
|
| 10 |
pygettext uses Python's standard tokenize module to scan Python source |
| 11 |
code, generating .pot files identical to what GNU xgettext[2] generates |
| 12 |
for C and C++ code. From there, the standard GNU tools can be used. |
| 13 |
|
| 14 |
A word about marking Python strings as candidates for translation. GNU |
| 15 |
xgettext recognizes the following keywords: gettext, dgettext, dcgettext, |
| 16 |
and gettext_noop. But those can be a lot of text to include all over your |
| 17 |
code. C and C++ have a trick: they use the C preprocessor. Most |
| 18 |
internationalized C source includes a #define for gettext() to _() so that |
| 19 |
what has to be written in the source is much less. Thus these are both |
| 20 |
translatable strings: |
| 21 |
|
| 22 |
gettext("Translatable String") |
| 23 |
_("Translatable String") |
| 24 |
|
| 25 |
Python of course has no preprocessor so this doesn't work so well. Thus, |
| 26 |
pygettext searches only for _() by default, but see the -k/--keyword flag |
| 27 |
below for how to augment this. |
| 28 |
|
| 29 |
[1] https://www.python.org/workshops/1997-10/proceedings/loewis.html |
| 30 |
[2] https://www.gnu.org/software/gettext/gettext.html |
| 31 |
|
| 32 |
NOTE: pygettext attempts to be option and feature compatible with GNU |
| 33 |
xgettext where ever possible. However some options are still missing or are |
| 34 |
not fully implemented. Also, xgettext's use of command line switches with |
| 35 |
option arguments is broken, and in these cases, pygettext just defines |
| 36 |
additional switches. |
| 37 |
|
| 38 |
NOTE: The public interface of pygettext is limited to the command-line |
| 39 |
interface only. The internal API is subject to change without notice. |
| 40 |
|
| 41 |
Usage: pygettext [options] inputfile ... |
| 42 |
|
| 43 |
Options: |
| 44 |
|
| 45 |
-a |
| 46 |
--extract-all |
| 47 |
Deprecated: Not implemented and will be removed in a future version. |
| 48 |
|
| 49 |
-cTAG |
| 50 |
--add-comments=TAG |
| 51 |
Extract translator comments. Comments must start with TAG and |
| 52 |
must precede the gettext call. Multiple -cTAG options are allowed. |
| 53 |
In that case, any comment matching any of the TAGs will be extracted. |
| 54 |
|
| 55 |
-d name |
| 56 |
--default-domain=name |
| 57 |
Rename the default output file from messages.pot to name.pot. |
| 58 |
|
| 59 |
-E |
| 60 |
--escape |
| 61 |
Replace non-ASCII characters with octal escape sequences. |
| 62 |
|
| 63 |
-D |
| 64 |
--docstrings |
| 65 |
Extract module, class, method, and function docstrings. These do |
| 66 |
not need to be wrapped in _() markers, and in fact cannot be for |
| 67 |
Python to consider them docstrings. (See also the -X option). |
| 68 |
|
| 69 |
-h |
| 70 |
--help |
| 71 |
Print this help message and exit. |
| 72 |
|
| 73 |
-k word |
| 74 |
--keyword=word |
| 75 |
Keywords to look for in addition to the default set, which are: |
| 76 |
_, gettext, ngettext, pgettext, npgettext, dgettext, dngettext, |
| 77 |
dpgettext, and dnpgettext. |
| 78 |
|
| 79 |
You can have multiple -k flags on the command line. |
| 80 |
|
| 81 |
-K |
| 82 |
--no-default-keywords |
| 83 |
Disable the default set of keywords (see above). Any keywords |
| 84 |
explicitly added with the -k/--keyword option are still recognized. |
| 85 |
|
| 86 |
--no-location |
| 87 |
Do not write filename/lineno location comments. |
| 88 |
|
| 89 |
-n |
| 90 |
--add-location |
| 91 |
Write filename/lineno location comments indicating where each |
| 92 |
extracted string is found in the source. These lines appear before |
| 93 |
each msgid. The style of comments is controlled by the -S/--style |
| 94 |
option. This is the default. |
| 95 |
|
| 96 |
-o filename |
| 97 |
--output=filename |
| 98 |
Rename the default output file from messages.pot to filename. If |
| 99 |
filename is `-' then the output is sent to standard out. |
| 100 |
|
| 101 |
-p dir |
| 102 |
--output-dir=dir |
| 103 |
Output files will be placed in directory dir. |
| 104 |
|
| 105 |
-S stylename |
| 106 |
--style stylename |
| 107 |
Specify which style to use for location comments. Two styles are |
| 108 |
supported: |
| 109 |
|
| 110 |
Solaris # File: filename, line: line-number |
| 111 |
GNU #: filename:line |
| 112 |
|
| 113 |
The style name is case insensitive. GNU style is the default. |
| 114 |
|
| 115 |
-v |
| 116 |
--verbose |
| 117 |
Print the names of the files being processed. |
| 118 |
|
| 119 |
-V |
| 120 |
--version |
| 121 |
Print the version of pygettext and exit. |
| 122 |
|
| 123 |
-w columns |
| 124 |
--width=columns |
| 125 |
Set width of output to columns. |
| 126 |
|
| 127 |
-x filename |
| 128 |
--exclude-file=filename |
| 129 |
Specify a file that contains a list of strings that are not be |
| 130 |
extracted from the input files. Each string to be excluded must |
| 131 |
appear on a line by itself in the file. |
| 132 |
|
| 133 |
-X filename |
| 134 |
--no-docstrings=filename |
| 135 |
Specify a file that contains a list of files (one per line) that |
| 136 |
should not have their docstrings extracted. This is only useful in |
| 137 |
conjunction with the -D option above. |
| 138 |
|
| 139 |
If `inputfile' is -, standard input is read. |
| 140 |
""" |
| 141 |
|
| 142 |
import ast |
| 143 |
import getopt |
| 144 |
import glob |
| 145 |
import importlib.machinery |
| 146 |
import importlib.util |
| 147 |
import os |
| 148 |
import sys |
| 149 |
import time |
| 150 |
import tokenize |
| 151 |
from dataclasses import dataclass, field |
| 152 |
from io import BytesIO |
| 153 |
from operator import itemgetter |
| 154 |
|
| 155 |
__version__ = '1.5' |
| 156 |
|
| 157 |
|
| 158 |
# The normal pot-file header. msgmerge and Emacs's po-mode work better if it's |
| 159 |
# there. |
| 160 |
pot_header = '''\ |
| 161 |
# SOME DESCRIPTIVE TITLE. |
| 162 |
# Copyright (C) YEAR ORGANIZATION |
| 163 |
# FIRST AUTHOR <EMAIL@ADDRESS>, YEAR. |
| 164 |
# |
| 165 |
msgid "" |
| 166 |
msgstr "" |
| 167 |
"Project-Id-Version: PACKAGE VERSION\\n" |
| 168 |
"POT-Creation-Date: %(time)s\\n" |
| 169 |
"PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\\n" |
| 170 |
"Last-Translator: FULL NAME <EMAIL@ADDRESS>\\n" |
| 171 |
"Language-Team: LANGUAGE <[email protected]>\\n" |
| 172 |
"MIME-Version: 1.0\\n" |
| 173 |
"Content-Type: text/plain; charset=%(charset)s\\n" |
| 174 |
"Content-Transfer-Encoding: %(encoding)s\\n" |
| 175 |
"Generated-By: pygettext.py %(version)s\\n" |
| 176 |
|
| 177 |
''' |
| 178 |
|
| 179 |
|
| 180 |
def usage(code, msg=''): |
| 181 |
print(__doc__, file=sys.stderr) |
| 182 |
if msg: |
| 183 |
print(msg, file=sys.stderr) |
| 184 |
sys.exit(code) |
| 185 |
|
| 186 |
|
| 187 |
def make_escapes(pass_nonascii): |
| 188 |
global escapes, escape |
| 189 |
if pass_nonascii: |
| 190 |
# Allow non-ascii characters to pass through so that e.g. 'msgid |
| 191 |
# "Höhe"' would not result in 'msgid "H\366he"'. Otherwise we |
| 192 |
# escape any character outside the 32..126 range. |
| 193 |
escape = escape_ascii |
| 194 |
else: |
| 195 |
escape = escape_nonascii |
| 196 |
escapes = [fr"\{i:03o}" for i in range(256)] |
| 197 |
for i in range(32, 127): |
| 198 |
escapes[i] = chr(i) |
| 199 |
escapes[ord('\\')] = r'\\' |
| 200 |
escapes[ord('\t')] = r'\t' |
| 201 |
escapes[ord('\r')] = r'\r' |
| 202 |
escapes[ord('\n')] = r'\n' |
| 203 |
escapes[ord('\"')] = r'\"' |
| 204 |
|
| 205 |
|
| 206 |
def escape_ascii(s, encoding): |
| 207 |
return ''.join(escapes[ord(c)] if ord(c) < 128 else c |
| 208 |
if c.isprintable() else escape_nonascii(c, encoding) |
| 209 |
for c in s) |
| 210 |
|
| 211 |
|
| 212 |
def escape_nonascii(s, encoding): |
| 213 |
return ''.join(escapes[b] for b in s.encode(encoding)) |
| 214 |
|
| 215 |
|
| 216 |
def normalize(s, encoding): |
| 217 |
# This converts the various Python string types into a format that is |
| 218 |
# appropriate for .po files, namely much closer to C style. |
| 219 |
lines = s.split('\n') |
| 220 |
if len(lines) == 1: |
| 221 |
s = '"' + escape(s, encoding) + '"' |
| 222 |
else: |
| 223 |
if not lines[-1]: |
| 224 |
del lines[-1] |
| 225 |
lines[-1] = lines[-1] + '\n' |
| 226 |
for i in range(len(lines)): |
| 227 |
lines[i] = escape(lines[i], encoding) |
| 228 |
lineterm = '\\n"\n"' |
| 229 |
s = '""\n"' + lineterm.join(lines) + '"' |
| 230 |
return s |
| 231 |
|
| 232 |
|
| 233 |
def containsAny(str, set): |
| 234 |
"""Check whether 'str' contains ANY of the chars in 'set'""" |
| 235 |
return 1 in [c in str for c in set] |
| 236 |
|
| 237 |
|
| 238 |
def getFilesForName(name): |
| 239 |
"""Get a list of module files for a filename, a module or package name, |
| 240 |
or a directory. |
| 241 |
""" |
| 242 |
if not os.path.exists(name): |
| 243 |
# check for glob chars |
| 244 |
if containsAny(name, "*?[]"): |
| 245 |
files = glob.glob(name) |
| 246 |
list = [] |
| 247 |
for file in files: |
| 248 |
list.extend(getFilesForName(file)) |
| 249 |
return list |
| 250 |
|
| 251 |
# try to find module or package |
| 252 |
try: |
| 253 |
spec = importlib.util.find_spec(name) |
| 254 |
name = spec.origin |
| 255 |
except ImportError: |
| 256 |
name = None |
| 257 |
if not name: |
| 258 |
return [] |
| 259 |
|
| 260 |
if os.path.isdir(name): |
| 261 |
# find all python files in directory |
| 262 |
list = [] |
| 263 |
# get extension for python source files |
| 264 |
_py_ext = importlib.machinery.SOURCE_SUFFIXES[0] |
| 265 |
for root, dirs, files in os.walk(name): |
| 266 |
# don't recurse into CVS directories |
| 267 |
if 'CVS' in dirs: |
| 268 |
dirs.remove('CVS') |
| 269 |
# add all *.py files to list |
| 270 |
list.extend( |
| 271 |
[os.path.join(root, file) for file in files |
| 272 |
if os.path.splitext(file)[1] == _py_ext] |
| 273 |
) |
| 274 |
return list |
| 275 |
elif os.path.exists(name): |
| 276 |
# a single file |
| 277 |
return [name] |
| 278 |
|
| 279 |
return [] |
| 280 |
|
| 281 |
|
| 282 |
# Key is the function name, value is a dictionary mapping argument positions to the |
| 283 |
# type of the argument. The type is one of 'msgid', 'msgid_plural', or 'msgctxt'. |
| 284 |
DEFAULTKEYWORDS = { |
| 285 |
'_': {'msgid': 0}, |
| 286 |
'gettext': {'msgid': 0}, |
| 287 |
'ngettext': {'msgid': 0, 'msgid_plural': 1}, |
| 288 |
'pgettext': {'msgctxt': 0, 'msgid': 1}, |
| 289 |
'npgettext': {'msgctxt': 0, 'msgid': 1, 'msgid_plural': 2}, |
| 290 |
'dgettext': {'msgid': 1}, |
| 291 |
'dngettext': {'msgid': 1, 'msgid_plural': 2}, |
| 292 |
'dpgettext': {'msgctxt': 1, 'msgid': 2}, |
| 293 |
'dnpgettext': {'msgctxt': 1, 'msgid': 2, 'msgid_plural': 3}, |
| 294 |
} |
| 295 |
|
| 296 |
|
| 297 |
def parse_spec(spec): |
| 298 |
"""Parse a keyword spec string into a dictionary. |
| 299 |
|
| 300 |
The keyword spec format defines the name of the gettext function and the |
| 301 |
positions of the arguments that correspond to msgid, msgid_plural, and |
| 302 |
msgctxt. The format is as follows: |
| 303 |
|
| 304 |
name - the name of the gettext function, assumed to |
| 305 |
have a single argument that is the msgid. |
| 306 |
name:pos1 - the name of the gettext function and the position |
| 307 |
of the msgid argument. |
| 308 |
name:pos1,pos2 - the name of the gettext function and the positions |
| 309 |
of the msgid and msgid_plural arguments. |
| 310 |
name:pos1,pos2c - the name of the gettext function and the positions |
| 311 |
of the msgid and msgctxt arguments. |
| 312 |
name:pos1,pos2,pos3c - the name of the gettext function and the |
| 313 |
positions of the msgid, msgid_plural, and |
| 314 |
msgctxt arguments. |
| 315 |
|
| 316 |
As an example, the spec 'foo:1,2,3c' means that the function foo has three |
| 317 |
arguments, the first one is the msgid, the second one is the msgid_plural, |
| 318 |
and the third one is the msgctxt. The positions are 1-based. |
| 319 |
|
| 320 |
The msgctxt argument can appear in any position, but it can only appear |
| 321 |
once. For example, the keyword specs 'foo:3c,1,2' and 'foo:1,2,3c' are |
| 322 |
equivalent. |
| 323 |
|
| 324 |
See https://www.gnu.org/software/gettext/manual/gettext.html |
| 325 |
for more information. |
| 326 |
""" |
| 327 |
parts = spec.strip().split(':', 1) |
| 328 |
if len(parts) == 1: |
| 329 |
name = parts[0] |
| 330 |
return name, {'msgid': 0} |
| 331 |
|
| 332 |
name, args = parts |
| 333 |
if not args: |
| 334 |
raise ValueError(f'Invalid keyword spec {spec!r}: ' |
| 335 |
'missing argument positions') |
| 336 |
|
| 337 |
result = {} |
| 338 |
for arg in args.split(','): |
| 339 |
arg = arg.strip() |
| 340 |
is_context = False |
| 341 |
if arg.endswith('c'): |
| 342 |
is_context = True |
| 343 |
arg = arg[:-1] |
| 344 |
|
| 345 |
try: |
| 346 |
pos = int(arg) - 1 |
| 347 |
except ValueError as e: |
| 348 |
raise ValueError(f'Invalid keyword spec {spec!r}: ' |
| 349 |
'position is not an integer') from e |
| 350 |
|
| 351 |
if pos < 0: |
| 352 |
raise ValueError(f'Invalid keyword spec {spec!r}: ' |
| 353 |
'argument positions must be strictly positive') |
| 354 |
|
| 355 |
if pos in result.values(): |
| 356 |
raise ValueError(f'Invalid keyword spec {spec!r}: ' |
| 357 |
'duplicate positions') |
| 358 |
|
| 359 |
if is_context: |
| 360 |
if 'msgctxt' in result: |
| 361 |
raise ValueError(f'Invalid keyword spec {spec!r}: ' |
| 362 |
'msgctxt can only appear once') |
| 363 |
result['msgctxt'] = pos |
| 364 |
elif 'msgid' not in result: |
| 365 |
result['msgid'] = pos |
| 366 |
elif 'msgid_plural' not in result: |
| 367 |
result['msgid_plural'] = pos |
| 368 |
else: |
| 369 |
raise ValueError(f'Invalid keyword spec {spec!r}: ' |
| 370 |
'too many positions') |
| 371 |
|
| 372 |
if 'msgid' not in result and 'msgctxt' in result: |
| 373 |
raise ValueError(f'Invalid keyword spec {spec!r}: ' |
| 374 |
'msgctxt cannot appear without msgid') |
| 375 |
|
| 376 |
return name, result |
| 377 |
|
| 378 |
|
| 379 |
def unparse_spec(name, spec): |
| 380 |
"""Unparse a keyword spec dictionary into a string.""" |
| 381 |
if spec == {'msgid': 0}: |
| 382 |
return name |
| 383 |
|
| 384 |
parts = [] |
| 385 |
for arg, pos in sorted(spec.items(), key=lambda x: x[1]): |
| 386 |
if arg == 'msgctxt': |
| 387 |
parts.append(f'{pos + 1}c') |
| 388 |
else: |
| 389 |
parts.append(str(pos + 1)) |
| 390 |
return f'{name}:{','.join(parts)}' |
| 391 |
|
| 392 |
|
| 393 |
def process_keywords(keywords, *, no_default_keywords): |
| 394 |
custom_keywords = {} |
| 395 |
for spec in dict.fromkeys(keywords): |
| 396 |
name, spec = parse_spec(spec) |
| 397 |
if name not in custom_keywords: |
| 398 |
custom_keywords[name] = [] |
| 399 |
custom_keywords[name].append(spec) |
| 400 |
|
| 401 |
if no_default_keywords: |
| 402 |
return custom_keywords |
| 403 |
|
| 404 |
# custom keywords override default keywords |
| 405 |
for name, spec in DEFAULTKEYWORDS.items(): |
| 406 |
if name not in custom_keywords: |
| 407 |
custom_keywords[name] = [] |
| 408 |
if spec not in custom_keywords[name]: |
| 409 |
custom_keywords[name].append(spec) |
| 410 |
return custom_keywords |
| 411 |
|
| 412 |
|
| 413 |
@dataclass(frozen=True) |
| 414 |
class Location: |
| 415 |
filename: str |
| 416 |
lineno: int |
| 417 |
|
| 418 |
def __lt__(self, other): |
| 419 |
return (self.filename, self.lineno) < (other.filename, other.lineno) |
| 420 |
|
| 421 |
|
| 422 |
@dataclass |
| 423 |
class Message: |
| 424 |
msgid: str |
| 425 |
msgid_plural: str | None |
| 426 |
msgctxt: str | None |
| 427 |
locations: set[Location] = field(default_factory=set) |
| 428 |
is_docstring: bool = False |
| 429 |
comments: list[str] = field(default_factory=list) |
| 430 |
|
| 431 |
def add_location(self, filename, lineno, msgid_plural=None, *, |
| 432 |
is_docstring=False, comments=None): |
| 433 |
if self.msgid_plural is None: |
| 434 |
self.msgid_plural = msgid_plural |
| 435 |
self.locations.add(Location(filename, lineno)) |
| 436 |
self.is_docstring |= is_docstring |
| 437 |
if comments: |
| 438 |
self.comments.extend(comments) |
| 439 |
|
| 440 |
|
| 441 |
def get_source_comments(source): |
| 442 |
""" |
| 443 |
Return a dictionary mapping line numbers to |
| 444 |
comments in the source code. |
| 445 |
""" |
| 446 |
comments = {} |
| 447 |
for token in tokenize.tokenize(BytesIO(source).readline): |
| 448 |
if token.type == tokenize.COMMENT: |
| 449 |
# Remove any leading combination of '#' and whitespace |
| 450 |
comment = token.string.lstrip('# \t') |
| 451 |
comments[token.start[0]] = comment |
| 452 |
return comments |
| 453 |
|
| 454 |
|
| 455 |
class GettextVisitor(ast.NodeVisitor): |
| 456 |
def __init__(self, options): |
| 457 |
super().__init__() |
| 458 |
self.options = options |
| 459 |
self.filename = None |
| 460 |
self.messages = {} |
| 461 |
self.comments = {} |
| 462 |
|
| 463 |
def visit_file(self, source, filename): |
| 464 |
try: |
| 465 |
module_tree = ast.parse(source) |
| 466 |
except SyntaxError: |
| 467 |
return |
| 468 |
|
| 469 |
self.filename = filename |
| 470 |
if self.options.comment_tags: |
| 471 |
self.comments = get_source_comments(source) |
| 472 |
self.visit(module_tree) |
| 473 |
|
| 474 |
def visit_Module(self, node): |
| 475 |
self._extract_docstring(node) |
| 476 |
self.generic_visit(node) |
| 477 |
|
| 478 |
visit_ClassDef = visit_FunctionDef = visit_AsyncFunctionDef = visit_Module |
| 479 |
|
| 480 |
def visit_Call(self, node): |
| 481 |
self._extract_message(node) |
| 482 |
self.generic_visit(node) |
| 483 |
|
| 484 |
def _extract_docstring(self, node): |
| 485 |
if (not self.options.docstrings or |
| 486 |
self.options.nodocstrings.get(self.filename)): |
| 487 |
return |
| 488 |
|
| 489 |
docstring = ast.get_docstring(node) |
| 490 |
if docstring is not None: |
| 491 |
lineno = node.body[0].lineno # The first statement is the docstring |
| 492 |
self._add_message(lineno, docstring, is_docstring=True) |
| 493 |
|
| 494 |
def _extract_message(self, node): |
| 495 |
func_name = self._get_func_name(node) |
| 496 |
errors = [] |
| 497 |
specs = self.options.keywords.get(func_name, []) |
| 498 |
for spec in specs: |
| 499 |
err = self._extract_message_with_spec(node, spec) |
| 500 |
if err is None: |
| 501 |
return |
| 502 |
errors.append(err) |
| 503 |
|
| 504 |
if not errors: |
| 505 |
return |
| 506 |
if len(errors) == 1: |
| 507 |
print(f'*** {self.filename}:{node.lineno}: {errors[0]}', |
| 508 |
file=sys.stderr) |
| 509 |
else: |
| 510 |
# There are multiple keyword specs for the function name and |
| 511 |
# none of them could be extracted. Print a general error |
| 512 |
# message and list the errors for each keyword spec. |
| 513 |
print(f'*** {self.filename}:{node.lineno}: ' |
| 514 |
f'No keywords matched gettext call "{func_name}":', |
| 515 |
file=sys.stderr) |
| 516 |
for spec, err in zip(specs, errors, strict=True): |
| 517 |
unparsed = unparse_spec(func_name, spec) |
| 518 |
print(f'\tkeyword="{unparsed}": {err}', file=sys.stderr) |
| 519 |
|
| 520 |
def _extract_message_with_spec(self, node, spec): |
| 521 |
"""Extract a gettext call with the given spec. |
| 522 |
|
| 523 |
Return None if the gettext call was successfully extracted, |
| 524 |
otherwise return an error message. |
| 525 |
""" |
| 526 |
max_index = max(spec.values()) |
| 527 |
has_var_positional = any(isinstance(arg, ast.Starred) for |
| 528 |
arg in node.args[:max_index+1]) |
| 529 |
if has_var_positional: |
| 530 |
return ('Variable positional arguments are not ' |
| 531 |
'allowed in gettext calls') |
| 532 |
|
| 533 |
if max_index >= len(node.args): |
| 534 |
return (f'Expected at least {max_index + 1} positional ' |
| 535 |
f'argument(s) in gettext call, got {len(node.args)}') |
| 536 |
|
| 537 |
msg_data = {} |
| 538 |
for arg_type, position in spec.items(): |
| 539 |
arg = node.args[position] |
| 540 |
if not self._is_string_const(arg): |
| 541 |
return (f'Expected a string constant for argument ' |
| 542 |
f'{position + 1}, got {ast.unparse(arg)}') |
| 543 |
msg_data[arg_type] = arg.value |
| 544 |
|
| 545 |
lineno = node.lineno |
| 546 |
comments = self._extract_comments(node) |
| 547 |
self._add_message(lineno, **msg_data, comments=comments) |
| 548 |
|
| 549 |
def _extract_comments(self, node): |
| 550 |
"""Extract translator comments. |
| 551 |
|
| 552 |
Translator comments must precede the gettext call and |
| 553 |
start with one of the comment prefixes defined by |
| 554 |
--add-comments=TAG. See the tests for examples. |
| 555 |
""" |
| 556 |
if not self.options.comment_tags: |
| 557 |
return [] |
| 558 |
|
| 559 |
comments = [] |
| 560 |
lineno = node.lineno - 1 |
| 561 |
# Collect an unbroken sequence of comments starting from |
| 562 |
# the line above the gettext call. |
| 563 |
while lineno >= 1: |
| 564 |
comment = self.comments.get(lineno) |
| 565 |
if comment is None: |
| 566 |
break |
| 567 |
comments.append(comment) |
| 568 |
lineno -= 1 |
| 569 |
|
| 570 |
# Find the first translator comment in the sequence and |
| 571 |
# return all comments starting from that comment. |
| 572 |
comments = comments[::-1] |
| 573 |
first_index = next((i for i, comment in enumerate(comments) |
| 574 |
if self._is_translator_comment(comment)), None) |
| 575 |
if first_index is None: |
| 576 |
return [] |
| 577 |
return comments[first_index:] |
| 578 |
|
| 579 |
def _is_translator_comment(self, comment): |
| 580 |
return comment.startswith(self.options.comment_tags) |
| 581 |
|
| 582 |
def _add_message( |
| 583 |
self, lineno, msgid, msgid_plural=None, msgctxt=None, *, |
| 584 |
is_docstring=False, comments=None): |
| 585 |
if msgid in self.options.toexclude: |
| 586 |
return |
| 587 |
|
| 588 |
if not comments: |
| 589 |
comments = [] |
| 590 |
|
| 591 |
key = self._key_for(msgid, msgctxt) |
| 592 |
message = self.messages.get(key) |
| 593 |
if message: |
| 594 |
message.add_location( |
| 595 |
self.filename, |
| 596 |
lineno, |
| 597 |
msgid_plural, |
| 598 |
is_docstring=is_docstring, |
| 599 |
comments=comments, |
| 600 |
) |
| 601 |
else: |
| 602 |
self.messages[key] = Message( |
| 603 |
msgid=msgid, |
| 604 |
msgid_plural=msgid_plural, |
| 605 |
msgctxt=msgctxt, |
| 606 |
locations={Location(self.filename, lineno)}, |
| 607 |
is_docstring=is_docstring, |
| 608 |
comments=comments, |
| 609 |
) |
| 610 |
|
| 611 |
@staticmethod |
| 612 |
def _key_for(msgid, msgctxt=None): |
| 613 |
if msgctxt is not None: |
| 614 |
return (msgctxt, msgid) |
| 615 |
return msgid |
| 616 |
|
| 617 |
def _get_func_name(self, node): |
| 618 |
match node.func: |
| 619 |
case ast.Name(id=id): |
| 620 |
return id |
| 621 |
case ast.Attribute(attr=attr): |
| 622 |
return attr |
| 623 |
case _: |
| 624 |
return None |
| 625 |
|
| 626 |
def _is_string_const(self, node): |
| 627 |
return isinstance(node, ast.Constant) and isinstance(node.value, str) |
| 628 |
|
| 629 |
def write_pot_file(messages, options, fp): |
| 630 |
timestamp = time.strftime('%Y-%m-%d %H:%M%z') |
| 631 |
encoding = fp.encoding if fp.encoding else 'UTF-8' |
| 632 |
print(pot_header % {'time': timestamp, 'version': __version__, |
| 633 |
'charset': encoding, |
| 634 |
'encoding': '8bit'}, file=fp) |
| 635 |
|
| 636 |
# Sort locations within each message by filename and lineno |
| 637 |
sorted_keys = [ |
| 638 |
(key, sorted(msg.locations)) |
| 639 |
for key, msg in messages.items() |
| 640 |
] |
| 641 |
# Sort messages by locations |
| 642 |
# For example, a message with locations [('test.py', 1), ('test.py', 2)] will |
| 643 |
# appear before a message with locations [('test.py', 1), ('test.py', 3)] |
| 644 |
sorted_keys.sort(key=itemgetter(1)) |
| 645 |
|
| 646 |
for key, locations in sorted_keys: |
| 647 |
msg = messages[key] |
| 648 |
|
| 649 |
for comment in msg.comments: |
| 650 |
print(f'#. {comment}', file=fp) |
| 651 |
|
| 652 |
if options.writelocations: |
| 653 |
# location comments are different b/w Solaris and GNU: |
| 654 |
if options.locationstyle == options.SOLARIS: |
| 655 |
for location in locations: |
| 656 |
print(f'# File: {location.filename}, line: {location.lineno}', file=fp) |
| 657 |
elif options.locationstyle == options.GNU: |
| 658 |
# fit as many locations on one line, as long as the |
| 659 |
# resulting line length doesn't exceed 'options.width' |
| 660 |
locline = '#:' |
| 661 |
for location in locations: |
| 662 |
s = f' {location.filename}:{location.lineno}' |
| 663 |
if len(locline) + len(s) <= options.width: |
| 664 |
locline = locline + s |
| 665 |
else: |
| 666 |
print(locline, file=fp) |
| 667 |
locline = f'#:{s}' |
| 668 |
if len(locline) > 2: |
| 669 |
print(locline, file=fp) |
| 670 |
if msg.is_docstring: |
| 671 |
# If the entry was gleaned out of a docstring, then add a |
| 672 |
# comment stating so. This is to aid translators who may wish |
| 673 |
# to skip translating some unimportant docstrings. |
| 674 |
print('#, docstring', file=fp) |
| 675 |
if msg.msgctxt is not None: |
| 676 |
print('msgctxt', normalize(msg.msgctxt, encoding), file=fp) |
| 677 |
print('msgid', normalize(msg.msgid, encoding), file=fp) |
| 678 |
if msg.msgid_plural is not None: |
| 679 |
print('msgid_plural', normalize(msg.msgid_plural, encoding), file=fp) |
| 680 |
print('msgstr[0] ""', file=fp) |
| 681 |
print('msgstr[1] ""\n', file=fp) |
| 682 |
else: |
| 683 |
print('msgstr ""\n', file=fp) |
| 684 |
|
| 685 |
|
| 686 |
def main(): |
| 687 |
try: |
| 688 |
opts, args = getopt.getopt( |
| 689 |
sys.argv[1:], |
| 690 |
'ac::d:DEhk:Kno:p:S:Vvw:x:X:', |
| 691 |
['extract-all', 'add-comments=?', 'default-domain=', 'escape', |
| 692 |
'help', 'keyword=', 'no-default-keywords', |
| 693 |
'add-location', 'no-location', 'output=', 'output-dir=', |
| 694 |
'style=', 'verbose', 'version', 'width=', 'exclude-file=', |
| 695 |
'docstrings', 'no-docstrings', |
| 696 |
]) |
| 697 |
except getopt.error as msg: |
| 698 |
usage(1, msg) |
| 699 |
|
| 700 |
# for holding option values |
| 701 |
class Options: |
| 702 |
# constants |
| 703 |
GNU = 1 |
| 704 |
SOLARIS = 2 |
| 705 |
# defaults |
| 706 |
extractall = 0 # FIXME: currently this option has no effect at all. |
| 707 |
escape = 0 |
| 708 |
keywords = [] |
| 709 |
outpath = '' |
| 710 |
outfile = 'messages.pot' |
| 711 |
writelocations = 1 |
| 712 |
locationstyle = GNU |
| 713 |
verbose = 0 |
| 714 |
width = 78 |
| 715 |
excludefilename = '' |
| 716 |
docstrings = 0 |
| 717 |
nodocstrings = {} |
| 718 |
comment_tags = set() |
| 719 |
|
| 720 |
options = Options() |
| 721 |
locations = {'gnu' : options.GNU, |
| 722 |
'solaris' : options.SOLARIS, |
| 723 |
} |
| 724 |
no_default_keywords = False |
| 725 |
# parse options |
| 726 |
for opt, arg in opts: |
| 727 |
if opt in ('-h', '--help'): |
| 728 |
usage(0) |
| 729 |
elif opt in ('-a', '--extract-all'): |
| 730 |
print("DeprecationWarning: -a/--extract-all is not implemented and will be removed in a future version", |
| 731 |
file=sys.stderr) |
| 732 |
options.extractall = 1 |
| 733 |
elif opt in ('-c', '--add-comments'): |
| 734 |
options.comment_tags.add(arg) |
| 735 |
elif opt in ('-d', '--default-domain'): |
| 736 |
options.outfile = arg + '.pot' |
| 737 |
elif opt in ('-E', '--escape'): |
| 738 |
options.escape = 1 |
| 739 |
elif opt in ('-D', '--docstrings'): |
| 740 |
options.docstrings = 1 |
| 741 |
elif opt in ('-k', '--keyword'): |
| 742 |
options.keywords.append(arg) |
| 743 |
elif opt in ('-K', '--no-default-keywords'): |
| 744 |
no_default_keywords = True |
| 745 |
elif opt in ('-n', '--add-location'): |
| 746 |
options.writelocations = 1 |
| 747 |
elif opt in ('--no-location',): |
| 748 |
options.writelocations = 0 |
| 749 |
elif opt in ('-S', '--style'): |
| 750 |
options.locationstyle = locations.get(arg.lower()) |
| 751 |
if options.locationstyle is None: |
| 752 |
usage(1, f'Invalid value for --style: {arg}') |
| 753 |
elif opt in ('-o', '--output'): |
| 754 |
options.outfile = arg |
| 755 |
elif opt in ('-p', '--output-dir'): |
| 756 |
options.outpath = arg |
| 757 |
elif opt in ('-v', '--verbose'): |
| 758 |
options.verbose = 1 |
| 759 |
elif opt in ('-V', '--version'): |
| 760 |
print(f'pygettext.py (xgettext for Python) {__version__}') |
| 761 |
sys.exit(0) |
| 762 |
elif opt in ('-w', '--width'): |
| 763 |
try: |
| 764 |
options.width = int(arg) |
| 765 |
except ValueError: |
| 766 |
usage(1, f'--width argument must be an integer: {arg}') |
| 767 |
elif opt in ('-x', '--exclude-file'): |
| 768 |
options.excludefilename = arg |
| 769 |
elif opt in ('-X', '--no-docstrings'): |
| 770 |
fp = open(arg) |
| 771 |
try: |
| 772 |
while 1: |
| 773 |
line = fp.readline() |
| 774 |
if not line: |
| 775 |
break |
| 776 |
options.nodocstrings[line[:-1]] = 1 |
| 777 |
finally: |
| 778 |
fp.close() |
| 779 |
|
| 780 |
options.comment_tags = tuple(options.comment_tags) |
| 781 |
|
| 782 |
# calculate escapes |
| 783 |
make_escapes(not options.escape) |
| 784 |
|
| 785 |
# calculate all keywords |
| 786 |
try: |
| 787 |
options.keywords = process_keywords( |
| 788 |
options.keywords, |
| 789 |
no_default_keywords=no_default_keywords) |
| 790 |
except ValueError as e: |
| 791 |
print(e, file=sys.stderr) |
| 792 |
sys.exit(1) |
| 793 |
|
| 794 |
# initialize list of strings to exclude |
| 795 |
if options.excludefilename: |
| 796 |
try: |
| 797 |
with open(options.excludefilename) as fp: |
| 798 |
options.toexclude = fp.readlines() |
| 799 |
except OSError: |
| 800 |
print(f"Can't read --exclude-file: {options.excludefilename}", |
| 801 |
file=sys.stderr) |
| 802 |
sys.exit(1) |
| 803 |
else: |
| 804 |
options.toexclude = [] |
| 805 |
|
| 806 |
# resolve args to module lists |
| 807 |
expanded = [] |
| 808 |
for arg in args: |
| 809 |
if arg == '-': |
| 810 |
expanded.append(arg) |
| 811 |
else: |
| 812 |
expanded.extend(getFilesForName(arg)) |
| 813 |
args = expanded |
| 814 |
|
| 815 |
# slurp through all the files |
| 816 |
visitor = GettextVisitor(options) |
| 817 |
for filename in args: |
| 818 |
if filename == '-': |
| 819 |
if options.verbose: |
| 820 |
print('Reading standard input') |
| 821 |
source = sys.stdin.buffer.read() |
| 822 |
else: |
| 823 |
if options.verbose: |
| 824 |
print(f'Working on {filename}') |
| 825 |
with open(filename, 'rb') as fp: |
| 826 |
source = fp.read() |
| 827 |
|
| 828 |
visitor.visit_file(source, filename) |
| 829 |
|
| 830 |
# write the output |
| 831 |
if options.outfile == '-': |
| 832 |
fp = sys.stdout |
| 833 |
closep = 0 |
| 834 |
else: |
| 835 |
if options.outpath: |
| 836 |
options.outfile = os.path.join(options.outpath, options.outfile) |
| 837 |
fp = open(options.outfile, 'w') |
| 838 |
closep = 1 |
| 839 |
try: |
| 840 |
write_pot_file(visitor.messages, options, fp) |
| 841 |
finally: |
| 842 |
if closep: |
| 843 |
fp.close() |
| 844 |
|
| 845 |
|
| 846 |
if __name__ == '__main__': |
| 847 |
main() |