Debian Codegraph @

setup/polib.py

raw
1 #
2 # License: MIT (see LICENSE file provided)
3 # vim: set expandtab tabstop=4 shiftwidth=4 softtabstop=4:
4
5 """
6 **polib** allows you to manipulate, create, modify gettext files (pot, po and
7 mo files). You can load existing files, iterate through it's entries, add,
8 modify entries, comments or metadata, etc. or create new po files from scratch.
9
10 **polib** provides a simple and pythonic API via the :func:`~polib.pofile` and
11 :func:`~polib.mofile` convenience functions.
12 """
13
14 import array
15 import codecs
16 import os
17 import re
18 import struct
19 import sys
20 import textwrap
21 import io
22
23
24 __author__ = 'David Jean Louis <[email protected]>'
25 __version__ = '1.2.0'
26 __all__ = ['pofile', 'POFile', 'POEntry', 'mofile', 'MOFile', 'MOEntry',
27 'default_encoding', 'escape', 'unescape', 'detect_encoding', ]
28
29
30 # the default encoding to use when encoding cannot be detected
31 default_encoding = 'utf-8'
32
33
34 # python 2/3 compatibility helpers {{{
35 PY3 = True
36 text_type = str
37
38 def b(s):
39 return s.encode()
40
41 def u(s):
42 return s
43 # }}}
44
45
46 # _pofile_or_mofile {{{
47 def _pofile_or_mofile(f, type, **kwargs):
48 """
49 Internal function used by :func:`polib.pofile` and :func:`polib.mofile` to
50 honor the DRY concept.
51 """
52 # get the file encoding
53 enc = kwargs.get('encoding')
54 if enc is None:
55 enc = detect_encoding(f, type == 'mofile')
56
57 # parse the file
58 kls = type == 'pofile' and _POFileParser or _MOFileParser
59 parser = kls(
60 f,
61 encoding=enc,
62 check_for_duplicates=kwargs.get('check_for_duplicates', False),
63 klass=kwargs.get('klass')
64 )
65 instance = parser.parse()
66 instance.wrapwidth = kwargs.get('wrapwidth', 78)
67 return instance
68 # }}}
69
70
71 # _is_file {{{
72 def _is_file(filename_or_contents):
73 """
74 Safely returns the value of os.path.exists(filename_or_contents).
75
76 Arguments:
77
78 ``filename_or_contents``
79 either a filename, or a string holding the contents of some file.
80 In the latter case, this function will always return False.
81 """
82 try:
83 return os.path.isfile(filename_or_contents)
84 except (TypeError, ValueError, UnicodeEncodeError):
85 return False
86 # }}}
87
88
89 # function pofile() {{{
90 def pofile(pofile, **kwargs):
91 """
92 Convenience function that parses the po or pot file ``pofile`` and returns
93 a :class:`~polib.POFile` instance.
94
95 Arguments:
96
97 ``pofile``
98 string, full or relative path to the po/pot file or its content (data).
99
100 ``wrapwidth``
101 integer, the wrap width, only useful when the ``-w`` option was passed
102 to xgettext (optional, default: ``78``).
103
104 ``encoding``
105 string, the encoding to use (e.g. "utf-8") (default: ``None``, the
106 encoding will be auto-detected).
107
108 ``check_for_duplicates``
109 whether to check for duplicate entries when adding entries to the
110 file (optional, default: ``False``).
111
112 ``klass``
113 class which is used to instantiate the return value (optional,
114 default: ``None``, the return value with be a :class:`~polib.POFile`
115 instance).
116 """
117 return _pofile_or_mofile(pofile, 'pofile', **kwargs)
118 # }}}
119
120
121 # function mofile() {{{
122 def mofile(mofile, **kwargs):
123 """
124 Convenience function that parses the mo file ``mofile`` and returns a
125 :class:`~polib.MOFile` instance.
126
127 Arguments:
128
129 ``mofile``
130 string, full or relative path to the mo file or its content (string
131 or bytes).
132
133 ``wrapwidth``
134 integer, the wrap width, only useful when the ``-w`` option was passed
135 to xgettext to generate the po file that was used to format the mo file
136 (optional, default: ``78``).
137
138 ``encoding``
139 string, the encoding to use (e.g. "utf-8") (default: ``None``, the
140 encoding will be auto-detected).
141
142 ``check_for_duplicates``
143 whether to check for duplicate entries when adding entries to the
144 file (optional, default: ``False``).
145
146 ``klass``
147 class which is used to instantiate the return value (optional,
148 default: ``None``, the return value with be a :class:`~polib.POFile`
149 instance).
150 """
151 return _pofile_or_mofile(mofile, 'mofile', **kwargs)
152 # }}}
153
154
155 # function detect_encoding() {{{
156 def detect_encoding(file, binary_mode=False):
157 """
158 Try to detect the encoding used by the ``file``. The ``file`` argument can
159 be a PO or MO file path or a string containing the contents of the file.
160 If the encoding cannot be detected, the function will return the value of
161 ``default_encoding``.
162
163 Arguments:
164
165 ``file``
166 string, full or relative path to the po/mo file or its content.
167
168 ``binary_mode``
169 boolean, set this to True if ``file`` is a mo file.
170 """
171 PATTERN = r'"?Content-Type:.+? charset=([\w_\-:\.]+)'
172 rxt = re.compile(u(PATTERN))
173 rxb = re.compile(b(PATTERN))
174
175 def charset_exists(charset):
176 """Check whether ``charset`` is valid or not."""
177 try:
178 codecs.lookup(charset)
179 except LookupError:
180 return False
181 return True
182
183 if not _is_file(file):
184 try:
185 match = rxt.search(file)
186 except TypeError:
187 match = rxb.search(file)
188 if match:
189 enc = match.group(1).strip()
190 if not isinstance(enc, text_type):
191 enc = enc.decode('utf-8')
192 if charset_exists(enc):
193 return enc
194 else:
195 # For PY3, always treat as binary
196 if binary_mode or PY3:
197 mode = 'rb'
198 rx = rxb
199 else:
200 mode = 'r'
201 rx = rxt
202 with open(file, mode) as f:
203 for line in f.readlines():
204 match = rx.search(line)
205 if match:
206 f.close()
207 enc = match.group(1).strip()
208 if not isinstance(enc, text_type):
209 enc = enc.decode('utf-8')
210 if charset_exists(enc):
211 return enc
212 return default_encoding
213 # }}}
214
215
216 # function escape() {{{
217 def escape(st):
218 """
219 Escapes the characters ``\\\\``, ``\\t``, ``\\n``, ``\\r``, ``\\v``,
220 ``\\b``, ``\\f`` and ``"`` in the given string ``st`` and returns it.
221 """
222 return st.replace('\\', r'\\')\
223 .replace('\t', r'\t')\
224 .replace('\r', r'\r')\
225 .replace('\n', r'\n')\
226 .replace('\v', r'\v')\
227 .replace('\b', r'\b')\
228 .replace('\f', r'\f')\
229 .replace('\"', r'\"')
230 # }}}
231
232
233 # function unescape() {{{
234 def unescape(st):
235 """
236 Unescapes the characters ``\\\\``, ``\\t``, ``\\n``, ``\\r``, ``\\v``,
237 ``\\b``, ``\\f`` and ``"`` in the given string ``st`` and returns it.
238 """
239 def unescape_repl(m):
240 m = m.group(1)
241 if m == 'n':
242 return '\n'
243 if m == 't':
244 return '\t'
245 if m == 'r':
246 return '\r'
247 if m == 'v':
248 return '\v'
249 if m == 'b':
250 return '\b'
251 if m == 'f':
252 return '\f'
253 if m == '\\':
254 return '\\'
255 return m # handles escaped double quote
256 return re.sub(r'\\(\\|n|t|r|v|b|f|")', unescape_repl, st)
257 # }}}
258
259
260 # function natural_sort() {{{
261 def natural_sort(lst):
262 """
263 Sort naturally the given list.
264 Credits: http://stackoverflow.com/a/4836734
265 """
266 def convert(text):
267 return int(text) if text.isdigit() else text.lower()
268
269 def alphanum_key(key):
270 return [convert(c) for c in re.split('([0-9]+)', key)]
271
272 return sorted(lst, key=alphanum_key)
273 # }}}
274
275
276 # class _BaseFile {{{
277 class _BaseFile(list):
278 """
279 Common base class for the :class:`~polib.POFile` and :class:`~polib.MOFile`
280 classes. This class should **not** be instantiated directly.
281 """
282
283 def __init__(self, *args, **kwargs):
284 """
285 Constructor, accepts the following keyword arguments:
286
287 ``pofile``
288 string, the path to the po or mo file, or its content as a string.
289
290 ``wrapwidth``
291 integer, the wrap width, only useful when the ``-w`` option was
292 passed to xgettext (optional, default: ``78``).
293
294 ``encoding``
295 string, the encoding to use, defaults to ``default_encoding``
296 global variable (optional).
297
298 ``check_for_duplicates``
299 whether to check for duplicate entries when adding entries to the
300 file, (optional, default: ``False``).
301 """
302 list.__init__(self)
303 # the opened file handle
304 pofile = kwargs.get('pofile', None)
305 if pofile and _is_file(pofile):
306 self.fpath = pofile
307 else:
308 self.fpath = kwargs.get('fpath')
309 # the width at which lines should be wrapped
310 self.wrapwidth = kwargs.get('wrapwidth', 78)
311 # the file encoding
312 self.encoding = kwargs.get('encoding', default_encoding)
313 # whether to check for duplicate entries or not
314 self.check_for_duplicates = kwargs.get('check_for_duplicates', False)
315 # header
316 self.header = ''
317 # both po and mo files have metadata
318 self.metadata = {}
319 self.metadata_is_fuzzy = 0
320
321 def __unicode__(self):
322 """
323 Returns the unicode representation of the file.
324 """
325 ret = []
326 entries = [self.metadata_as_entry()] + \
327 [e for e in self if not e.obsolete]
328 for entry in entries:
329 ret.append(entry.__unicode__(self.wrapwidth))
330 for entry in self.obsolete_entries():
331 ret.append(entry.__unicode__(self.wrapwidth))
332 ret = u('\n').join(ret)
333 return ret
334
335 if PY3:
336 def __str__(self):
337 return self.__unicode__()
338 else:
339 def __str__(self):
340 """
341 Returns the string representation of the file.
342 """
343 return unicode(self).encode(self.encoding)
344
345 def __contains__(self, entry):
346 """
347 Overridden ``list`` method to implement the membership test (in and
348 not in).
349 The method considers that an entry is in the file if it finds an entry
350 that has the same msgid (the test is **case sensitive**) and the same
351 msgctxt (or none for both entries).
352
353 Argument:
354
355 ``entry``
356 an instance of :class:`~polib._BaseEntry`.
357 """
358 return self.find(entry.msgid, by='msgid', msgctxt=entry.msgctxt) \
359 is not None
360
361 def __eq__(self, other):
362 return str(self) == str(other)
363
364 def append(self, entry):
365 """
366 Overridden method to check for duplicates entries, if a user tries to
367 add an entry that is already in the file, the method will raise a
368 ``ValueError`` exception.
369
370 Argument:
371
372 ``entry``
373 an instance of :class:`~polib._BaseEntry`.
374 """
375 # check_for_duplicates may not be defined (yet) when unpickling.
376 # But if pickling, we never want to check for duplicates anyway.
377 if getattr(self, 'check_for_duplicates', False) and entry in self:
378 raise ValueError('Entry "%s" already exists' % entry.msgid)
379 super().append(entry)
380
381 def insert(self, index, entry):
382 """
383 Overridden method to check for duplicates entries, if a user tries to
384 add an entry that is already in the file, the method will raise a
385 ``ValueError`` exception.
386
387 Arguments:
388
389 ``index``
390 index at which the entry should be inserted.
391
392 ``entry``
393 an instance of :class:`~polib._BaseEntry`.
394 """
395 if self.check_for_duplicates and entry in self:
396 raise ValueError('Entry "%s" already exists' % entry.msgid)
397 super().insert(index, entry)
398
399 def metadata_as_entry(self):
400 """
401 Returns the file metadata as a :class:`~polib.POFile` instance.
402 """
403 e = POEntry(msgid='')
404 mdata = self.ordered_metadata()
405 if mdata:
406 strs = []
407 for name, value in mdata:
408 # Strip whitespace off each line in a multi-line entry
409 strs.append(f'{name}: {value}')
410 e.msgstr = '\n'.join(strs) + '\n'
411 if self.metadata_is_fuzzy:
412 e.flags.append('fuzzy')
413 return e
414
415 def save(self, fpath=None, repr_method='__unicode__', newline=None):
416 """
417 Saves the po file to ``fpath``.
418 If it is an existing file and no ``fpath`` is provided, then the
419 existing file is rewritten with the modified data.
420
421 Keyword arguments:
422
423 ``fpath``
424 string, full or relative path to the file.
425
426 ``repr_method``
427 string, the method to use for output.
428
429 ``newline``
430 string, controls how universal newlines works
431 """
432 if self.fpath is None and fpath is None:
433 raise OSError('You must provide a file path to save() method')
434 contents = getattr(self, repr_method)()
435 if fpath is None:
436 fpath = self.fpath
437 if repr_method == 'to_binary':
438 with open(fpath, 'wb') as fhandle:
439 fhandle.write(contents)
440 else:
441 with open(
442 fpath,
443 'w',
444 encoding=self.encoding,
445 newline=newline
446 ) as fhandle:
447 if not isinstance(contents, text_type):
448 contents = contents.decode(self.encoding)
449 fhandle.write(contents)
450
451 # set the file path if not set
452 if self.fpath is None and fpath:
453 self.fpath = fpath
454
455 def find(self, st, by='msgid', include_obsolete_entries=False,
456 msgctxt=False):
457 """
458 Find the entry which msgid (or property identified by the ``by``
459 argument) matches the string ``st``.
460
461 Keyword arguments:
462
463 ``st``
464 string, the string to search for.
465
466 ``by``
467 string, the property to use for comparison (default: ``msgid``).
468
469 ``include_obsolete_entries``
470 boolean, whether to also search in entries that are obsolete.
471
472 ``msgctxt``
473 string, allows specifying a specific message context for the
474 search.
475 """
476 if include_obsolete_entries:
477 entries = self[:]
478 else:
479 entries = [e for e in self if not e.obsolete]
480 matches = []
481 for e in entries:
482 if getattr(e, by) == st:
483 if msgctxt is not False and e.msgctxt != msgctxt:
484 continue
485 matches.append(e)
486 if len(matches) == 1:
487 return matches[0]
488 elif len(matches) > 1:
489 if not msgctxt:
490 # find the entry with no msgctx
491 e = None
492 for m in matches:
493 if not m.msgctxt:
494 e = m
495 if e:
496 return e
497 # fallback to the first entry found
498 return matches[0]
499 return None
500
501 def ordered_metadata(self):
502 """
503 Convenience method that returns an ordered version of the metadata
504 dictionary. The return value is list of tuples (metadata name,
505 metadata_value).
506 """
507 # copy the dict first
508 metadata = self.metadata.copy()
509 data_order = [
510 'Project-Id-Version',
511 'Report-Msgid-Bugs-To',
512 'POT-Creation-Date',
513 'PO-Revision-Date',
514 'Last-Translator',
515 'Language-Team',
516 'Language',
517 'MIME-Version',
518 'Content-Type',
519 'Content-Transfer-Encoding',
520 'Plural-Forms'
521 ]
522 ordered_data = []
523 for data in data_order:
524 try:
525 value = metadata.pop(data)
526 ordered_data.append((data, value))
527 except KeyError:
528 pass
529 # the rest of the metadata will be alphabetically ordered since there
530 # are no specs for this AFAIK
531 for data in natural_sort(metadata.keys()):
532 value = metadata[data]
533 ordered_data.append((data, value))
534 return ordered_data
535
536 def to_binary(self):
537 """
538 Return the binary representation of the file.
539 """
540 offsets = []
541 entries = self.translated_entries()
542
543 # the keys are sorted in the .mo file
544 def cmp(_self, other):
545 # msgfmt compares entries with msgctxt if it exists
546 self_msgid = _self.msgctxt and _self.msgctxt or _self.msgid
547 other_msgid = other.msgctxt and other.msgctxt or other.msgid
548 if self_msgid > other_msgid:
549 return 1
550 elif self_msgid < other_msgid:
551 return -1
552 else:
553 return 0
554 # add metadata entry
555 entries.sort(key=lambda o: o.msgid_with_context.encode('utf-8'))
556 mentry = self.metadata_as_entry()
557 entries = [mentry] + entries
558 entries_len = len(entries)
559 ids, strs = b(''), b('')
560 for e in entries:
561 # For each string, we need size and file offset. Each string is
562 # NUL terminated; the NUL does not count into the size.
563 msgid = b('')
564 if e.msgctxt:
565 # Contexts are stored by storing the concatenation of the
566 # context, a <EOT> byte, and the original string
567 msgid = self._encode(e.msgctxt + '\4')
568 if e.msgid_plural:
569 msgstr = []
570 for index in sorted(e.msgstr_plural.keys()):
571 msgstr.append(e.msgstr_plural[index])
572 msgid += self._encode(e.msgid + '\0' + e.msgid_plural)
573 msgstr = self._encode('\0'.join(msgstr))
574 else:
575 msgid += self._encode(e.msgid)
576 msgstr = self._encode(e.msgstr)
577 offsets.append((len(ids), len(msgid), len(strs), len(msgstr)))
578 ids += msgid + b('\0')
579 strs += msgstr + b('\0')
580
581 # The header is 7 32-bit unsigned integers.
582 keystart = 7 * 4 + 16 * entries_len
583 # and the values start after the keys
584 valuestart = keystart + len(ids)
585 koffsets = []
586 voffsets = []
587 # The string table first has the list of keys, then the list of values.
588 # Each entry has first the size of the string, then the file offset.
589 for o1, l1, o2, l2 in offsets:
590 koffsets += [l1, o1 + keystart]
591 voffsets += [l2, o2 + valuestart]
592 offsets = koffsets + voffsets
593
594 output = struct.pack(
595 "Iiiiiii",
596 # Magic number
597 MOFile.MAGIC,
598 # Version
599 0,
600 # number of entries
601 entries_len,
602 # start of key index
603 7 * 4,
604 # start of value index
605 7 * 4 + entries_len * 8,
606 # size and offset of hash table, we don't use hash tables
607 0, keystart
608
609 )
610 if PY3 and sys.version_info.minor > 1: # python 3.2 or superior
611 output += array.array("i", offsets).tobytes()
612 else:
613 output += array.array("i", offsets).tostring()
614 output += ids
615 output += strs
616 return output
617
618 def _encode(self, mixed):
619 """
620 Encodes the given ``mixed`` argument with the file encoding if and
621 only if it's an unicode string and returns the encoded string.
622 """
623 if isinstance(mixed, text_type):
624 mixed = mixed.encode(self.encoding)
625 return mixed
626 # }}}
627
628
629 # class POFile {{{
630 class POFile(_BaseFile):
631 """
632 Po (or Pot) file reader/writer.
633 This class inherits the :class:`~polib._BaseFile` class and, by extension,
634 the python ``list`` type.
635 """
636
637 def __unicode__(self):
638 """
639 Returns the unicode representation of the po file.
640 """
641 ret, headers = '', self.header.split('\n')
642 for header in headers:
643 if not len(header):
644 ret += "#\n"
645 elif header[:1] in [',', ':']:
646 ret += '#%s\n' % header
647 else:
648 ret += '# %s\n' % header
649
650 if not isinstance(ret, text_type):
651 ret = ret.decode(self.encoding)
652
653 return ret + _BaseFile.__unicode__(self)
654
655 def save_as_mofile(self, fpath):
656 """
657 Saves the binary representation of the file to given ``fpath``.
658
659 Keyword argument:
660
661 ``fpath``
662 string, full or relative path to the mo file.
663 """
664 _BaseFile.save(self, fpath, 'to_binary')
665
666 def percent_translated(self):
667 """
668 Convenience method that returns the percentage of translated
669 messages.
670 """
671 total = len([e for e in self if not e.obsolete])
672 if total == 0:
673 return 100
674 translated = len(self.translated_entries())
675 return int(translated * 100 / float(total))
676
677 def translated_entries(self):
678 """
679 Convenience method that returns the list of translated entries.
680 """
681 return [e for e in self if e.translated()]
682
683 def untranslated_entries(self):
684 """
685 Convenience method that returns the list of untranslated entries.
686 """
687 return [e for e in self if not e.translated() and not e.obsolete
688 and not e.fuzzy]
689
690 def fuzzy_entries(self):
691 """
692 Convenience method that returns the list of fuzzy entries.
693 """
694 return [e for e in self if e.fuzzy and not e.obsolete]
695
696 def obsolete_entries(self):
697 """
698 Convenience method that returns the list of obsolete entries.
699 """
700 return [e for e in self if e.obsolete]
701
702 def merge(self, refpot):
703 """
704 Convenience method that merges the current pofile with the pot file
705 provided. It behaves exactly as the gettext msgmerge utility:
706
707 * comments of this file will be preserved, but extracted comments and
708 occurrences will be discarded;
709 * any translations or comments in the file will be discarded, however,
710 dot comments and file positions will be preserved;
711 * the fuzzy flags are preserved.
712
713 Keyword argument:
714
715 ``refpot``
716 object POFile, the reference catalog.
717 """
718 # Store entries in dict/set for faster access
719 self_entries = {
720 entry.msgid_with_context: entry for entry in self
721 }
722 refpot_msgids = {entry.msgid_with_context for entry in refpot}
723 # Merge entries that are in the refpot
724 for entry in refpot:
725 e = self_entries.get(entry.msgid_with_context)
726 if e is None:
727 e = POEntry()
728 self.append(e)
729 e.merge(entry)
730 # ok, now we must "obsolete" entries that are not in the refpot anymore
731 for entry in self:
732 if entry.msgid_with_context not in refpot_msgids:
733 entry.obsolete = True
734 # }}}
735
736
737 # class MOFile {{{
738 class MOFile(_BaseFile):
739 """
740 Mo file reader/writer.
741 This class inherits the :class:`~polib._BaseFile` class and, by
742 extension, the python ``list`` type.
743 """
744 MAGIC = 0x950412de
745 MAGIC_SWAPPED = 0xde120495
746
747 def __init__(self, *args, **kwargs):
748 """
749 Constructor, accepts all keywords arguments accepted by
750 :class:`~polib._BaseFile` class.
751 """
752 _BaseFile.__init__(self, *args, **kwargs)
753 self.magic_number = None
754 self.version = 0
755
756 def save_as_pofile(self, fpath):
757 """
758 Saves the mofile as a pofile to ``fpath``.
759
760 Keyword argument:
761
762 ``fpath``
763 string, full or relative path to the file.
764 """
765 _BaseFile.save(self, fpath)
766
767 def save(self, fpath=None):
768 """
769 Saves the mofile to ``fpath``.
770
771 Keyword argument:
772
773 ``fpath``
774 string, full or relative path to the file.
775 """
776 _BaseFile.save(self, fpath, 'to_binary')
777
778 def percent_translated(self):
779 """
780 Convenience method to keep the same interface with POFile instances.
781 """
782 return 100
783
784 def translated_entries(self):
785 """
786 Convenience method to keep the same interface with POFile instances.
787 """
788 return self
789
790 def untranslated_entries(self):
791 """
792 Convenience method to keep the same interface with POFile instances.
793 """
794 return []
795
796 def fuzzy_entries(self):
797 """
798 Convenience method to keep the same interface with POFile instances.
799 """
800 return []
801
802 def obsolete_entries(self):
803 """
804 Convenience method to keep the same interface with POFile instances.
805 """
806 return []
807 # }}}
808
809
810 # class _BaseEntry {{{
811 class _BaseEntry:
812 """
813 Base class for :class:`~polib.POEntry` and :class:`~polib.MOEntry` classes.
814 This class should **not** be instantiated directly.
815 """
816
817 def __init__(self, *args, **kwargs):
818 """
819 Constructor, accepts the following keyword arguments:
820
821 ``msgid``
822 string, the entry msgid.
823
824 ``msgstr``
825 string, the entry msgstr.
826
827 ``msgid_plural``
828 string, the entry msgid_plural.
829
830 ``msgstr_plural``
831 dict, the entry msgstr_plural lines.
832
833 ``msgctxt``
834 string, the entry context (msgctxt).
835
836 ``obsolete``
837 bool, whether the entry is "obsolete" or not.
838
839 ``encoding``
840 string, the encoding to use, defaults to ``default_encoding``
841 global variable (optional).
842 """
843 self.msgid = kwargs.get('msgid', '')
844 self.msgstr = kwargs.get('msgstr', '')
845 self.msgid_plural = kwargs.get('msgid_plural', '')
846 self.msgstr_plural = kwargs.get('msgstr_plural', {})
847 self.msgctxt = kwargs.get('msgctxt', None)
848 self.obsolete = kwargs.get('obsolete', False)
849 self.encoding = kwargs.get('encoding', default_encoding)
850
851 def __unicode__(self, wrapwidth=78):
852 """
853 Returns the unicode representation of the entry.
854 """
855 if self.obsolete:
856 delflag = '#~ '
857 else:
858 delflag = ''
859 ret = []
860 # write the msgctxt if any
861 if self.msgctxt is not None:
862 ret += self._str_field("msgctxt", delflag, "", self.msgctxt,
863 wrapwidth)
864 # write the msgid
865 ret += self._str_field("msgid", delflag, "", self.msgid, wrapwidth)
866 # write the msgid_plural if any
867 if self.msgid_plural:
868 ret += self._str_field("msgid_plural", delflag, "",
869 self.msgid_plural, wrapwidth)
870 if self.msgstr_plural:
871 # write the msgstr_plural if any
872 msgstrs = self.msgstr_plural
873 keys = list(msgstrs)
874 keys.sort()
875 for index in keys:
876 msgstr = msgstrs[index]
877 plural_index = '[%s]' % index
878 ret += self._str_field("msgstr", delflag, plural_index, msgstr,
879 wrapwidth)
880 else:
881 # otherwise write the msgstr
882 ret += self._str_field("msgstr", delflag, "", self.msgstr,
883 wrapwidth)
884 ret.append('')
885 ret = u('\n').join(ret)
886 return ret
887
888 if PY3:
889 def __str__(self):
890 return self.__unicode__()
891 else:
892 def __str__(self):
893 """
894 Returns the string representation of the entry.
895 """
896 return unicode(self).encode(self.encoding)
897
898 def __eq__(self, other):
899 return str(self) == str(other)
900
901 def _str_field(self, fieldname, delflag, plural_index, field,
902 wrapwidth=78):
903 lines = field.splitlines(True)
904 if len(lines) > 1:
905 lines = [''] + lines # start with initial empty line
906 else:
907 escaped_field = escape(field)
908 specialchars_count = 0
909 for c in ['\\', '\n', '\r', '\t', '\v', '\b', '\f', '"']:
910 specialchars_count += field.count(c)
911 # comparison must take into account fieldname length + one space
912 # + 2 quotes (eg. msgid "<string>")
913 flength = len(fieldname) + 3
914 if plural_index:
915 flength += len(plural_index)
916 real_wrapwidth = wrapwidth - flength + specialchars_count
917 if wrapwidth > 0 and len(field) > real_wrapwidth:
918 # Wrap the line but take field name into account
919 lines = [''] + [unescape(item) for item in textwrap.wrap(
920 escaped_field,
921 wrapwidth - 2, # 2 for quotes ""
922 drop_whitespace=False,
923 break_long_words=False
924 )]
925 else:
926 lines = [field]
927 if fieldname.startswith('previous_'):
928 # quick and dirty trick to get the real field name
929 fieldname = fieldname[9:]
930
931 ret = ['{}{}{} "{}"'.format(delflag, fieldname, plural_index,
932 escape(lines.pop(0)))]
933 for line in lines:
934 ret.append(f'{delflag}"{escape(line)}"')
935 return ret
936
937 @property
938 def msgid_with_context(self):
939 if self.msgctxt:
940 return '{}{}{}'.format(self.msgctxt, "\x04", self.msgid)
941 return self.msgid
942 # }}}
943
944
945 # class POEntry {{{
946 class POEntry(_BaseEntry):
947 """
948 Represents a po file entry.
949 """
950
951 def __init__(self, *args, **kwargs):
952 """
953 Constructor, accepts the following keyword arguments:
954
955 ``comment``
956 string, the entry comment.
957
958 ``tcomment``
959 string, the entry translator comment.
960
961 ``occurrences``
962 list, the entry occurrences.
963
964 ``flags``
965 list, the entry flags.
966
967 ``previous_msgctxt``
968 string, the entry previous context.
969
970 ``previous_msgid``
971 string, the entry previous msgid.
972
973 ``previous_msgid_plural``
974 string, the entry previous msgid_plural.
975
976 ``linenum``
977 integer, the line number of the entry
978 """
979 _BaseEntry.__init__(self, *args, **kwargs)
980 self.comment = kwargs.get('comment', '')
981 self.tcomment = kwargs.get('tcomment', '')
982 self.occurrences = kwargs.get('occurrences', [])
983 self.flags = kwargs.get('flags', [])
984 self.previous_msgctxt = kwargs.get('previous_msgctxt', None)
985 self.previous_msgid = kwargs.get('previous_msgid', None)
986 self.previous_msgid_plural = kwargs.get('previous_msgid_plural', None)
987 self.linenum = kwargs.get('linenum', None)
988
989 def __unicode__(self, wrapwidth=78):
990 """
991 Returns the unicode representation of the entry.
992 """
993 ret = []
994 # comments first, if any (with text wrapping as xgettext does)
995 if self.obsolete:
996 comments = [('tcomment', '# ')]
997 else:
998 comments = [('tcomment', '# '), ('comment', '#. ')]
999 for c in comments:
1000 val = getattr(self, c[0])
1001 if val:
1002 for comment in val.split('\n'):
1003 if wrapwidth > 0 and len(comment) + len(c[1]) > wrapwidth:
1004 ret += textwrap.wrap(
1005 comment,
1006 wrapwidth,
1007 initial_indent=c[1],
1008 subsequent_indent=c[1],
1009 break_long_words=False
1010 )
1011 else:
1012 ret.append(f'{c[1]}{comment}')
1013
1014 # occurrences (with text wrapping as xgettext does)
1015 if not self.obsolete and self.occurrences:
1016 filelist = []
1017 for fpath, lineno in self.occurrences:
1018 if lineno:
1019 filelist.append(f'{fpath}:{lineno}')
1020 else:
1021 filelist.append(fpath)
1022 filestr = ' '.join(filelist)
1023 if wrapwidth > 0 and len(filestr) + 3 > wrapwidth:
1024 # textwrap split words that contain hyphen, this is not
1025 # what we want for filenames, so the dirty hack is to
1026 # temporally replace hyphens with a char that a file cannot
1027 # contain, like "*"
1028 ret += [line.replace('*', '-') for line in textwrap.wrap(
1029 filestr.replace('-', '*'),
1030 wrapwidth,
1031 initial_indent='#: ',
1032 subsequent_indent='#: ',
1033 break_long_words=False
1034 )]
1035 else:
1036 ret.append('#: ' + filestr)
1037
1038 # flags (TODO: wrapping ?)
1039 if self.flags:
1040 ret.append('#, %s' % ', '.join(self.flags))
1041
1042 # previous context and previous msgid/msgid_plural
1043 fields = ['previous_msgctxt', 'previous_msgid',
1044 'previous_msgid_plural']
1045 if self.obsolete:
1046 prefix = "#~| "
1047 else:
1048 prefix = "#| "
1049 for f in fields:
1050 val = getattr(self, f)
1051 if val is not None:
1052 ret += self._str_field(f, prefix, "", val, wrapwidth)
1053
1054 ret.append(_BaseEntry.__unicode__(self, wrapwidth))
1055 ret = u('\n').join(ret)
1056 return ret
1057
1058 def __cmp__(self, other):
1059 """
1060 Called by comparison operations if rich comparison is not defined.
1061 """
1062 # First: Obsolete test
1063 if self.obsolete != other.obsolete:
1064 if self.obsolete:
1065 return -1
1066 else:
1067 return 1
1068 # Work on a copy to protect original
1069 occ1 = sorted(self.occurrences[:])
1070 occ2 = sorted(other.occurrences[:])
1071 if occ1 > occ2:
1072 return 1
1073 if occ1 < occ2:
1074 return -1
1075 # Compare context
1076 msgctxt = self.msgctxt or '0'
1077 othermsgctxt = other.msgctxt or '0'
1078 if msgctxt > othermsgctxt:
1079 return 1
1080 elif msgctxt < othermsgctxt:
1081 return -1
1082 # Compare msgid_plural
1083 msgid_plural = self.msgid_plural or '0'
1084 othermsgid_plural = other.msgid_plural or '0'
1085 if msgid_plural > othermsgid_plural:
1086 return 1
1087 elif msgid_plural < othermsgid_plural:
1088 return -1
1089 # Compare msgstr_plural
1090 if self.msgstr_plural and isinstance(self.msgstr_plural, dict):
1091 msgstr_plural = list(self.msgstr_plural.values())
1092 else:
1093 msgstr_plural = []
1094 if other.msgstr_plural and isinstance(other.msgstr_plural, dict):
1095 othermsgstr_plural = list(other.msgstr_plural.values())
1096 else:
1097 othermsgstr_plural = []
1098 if msgstr_plural > othermsgstr_plural:
1099 return 1
1100 elif msgstr_plural < othermsgstr_plural:
1101 return -1
1102 # Compare msgid
1103 if self.msgid > other.msgid:
1104 return 1
1105 elif self.msgid < other.msgid:
1106 return -1
1107 # Compare msgstr
1108 if self.msgstr > other.msgstr:
1109 return 1
1110 elif self.msgstr < other.msgstr:
1111 return -1
1112 return 0
1113
1114 def __gt__(self, other):
1115 return self.__cmp__(other) > 0
1116
1117 def __lt__(self, other):
1118 return self.__cmp__(other) < 0
1119
1120 def __ge__(self, other):
1121 return self.__cmp__(other) >= 0
1122
1123 def __le__(self, other):
1124 return self.__cmp__(other) <= 0
1125
1126 def __eq__(self, other):
1127 return self.__cmp__(other) == 0
1128
1129 def __ne__(self, other):
1130 return self.__cmp__(other) != 0
1131
1132 def translated(self):
1133 """
1134 Returns ``True`` if the entry has been translated or ``False``
1135 otherwise.
1136 """
1137 if self.obsolete or self.fuzzy:
1138 return False
1139 if self.msgstr != '':
1140 return True
1141 if self.msgstr_plural:
1142 for pos in self.msgstr_plural:
1143 if self.msgstr_plural[pos] == '':
1144 return False
1145 return True
1146 return False
1147
1148 def merge(self, other):
1149 """
1150 Merge the current entry with the given pot entry.
1151 """
1152 self.msgid = other.msgid
1153 self.msgctxt = other.msgctxt
1154 self.occurrences = other.occurrences
1155 self.comment = other.comment
1156 fuzzy = self.fuzzy
1157 self.flags = other.flags[:] # clone flags
1158 if fuzzy:
1159 self.flags.append('fuzzy')
1160 self.msgid_plural = other.msgid_plural
1161 self.obsolete = other.obsolete
1162 self.previous_msgctxt = other.previous_msgctxt
1163 self.previous_msgid = other.previous_msgid
1164 self.previous_msgid_plural = other.previous_msgid_plural
1165 if other.msgstr_plural:
1166 for pos in other.msgstr_plural:
1167 try:
1168 # keep existing translation at pos if any
1169 self.msgstr_plural[pos]
1170 except KeyError:
1171 self.msgstr_plural[pos] = ''
1172
1173 @property
1174 def fuzzy(self):
1175 return 'fuzzy' in self.flags
1176
1177 @fuzzy.setter
1178 def fuzzy(self, value):
1179 if value and not self.fuzzy:
1180 self.flags.insert(0, 'fuzzy')
1181 elif not value and self.fuzzy:
1182 self.flags.remove('fuzzy')
1183
1184 def __hash__(self):
1185 return hash((self.msgid, self.msgstr))
1186 # }}}
1187
1188
1189 # class MOEntry {{{
1190 class MOEntry(_BaseEntry):
1191 """
1192 Represents a mo file entry.
1193 """
1194 def __init__(self, *args, **kwargs):
1195 """
1196 Constructor, accepts the following keyword arguments,
1197 for consistency with :class:`~polib.POEntry`:
1198
1199 ``comment``
1200 ``tcomment``
1201 ``occurrences``
1202 ``flags``
1203 ``previous_msgctxt``
1204 ``previous_msgid``
1205 ``previous_msgid_plural``
1206
1207 Note: even though these keyword arguments are accepted,
1208 they hold no real meaning in the context of MO files
1209 and are simply ignored.
1210 """
1211 _BaseEntry.__init__(self, *args, **kwargs)
1212 self.comment = ''
1213 self.tcomment = ''
1214 self.occurrences = []
1215 self.flags = []
1216 self.previous_msgctxt = None
1217 self.previous_msgid = None
1218 self.previous_msgid_plural = None
1219
1220 def __hash__(self):
1221 return hash((self.msgid, self.msgstr))
1222 # }}}
1223
1224
1225 # class _POFileParser {{{
1226 class _POFileParser:
1227 """
1228 A finite state machine to efficiently and correctly parse po
1229 file format.
1230 """
1231
1232 def __init__(self, pofile, *args, **kwargs):
1233 """
1234 Constructor.
1235
1236 Keyword arguments:
1237
1238 ``pofile``
1239 string, path to the po file or its content
1240
1241 ``encoding``
1242 string, the encoding to use, defaults to ``default_encoding``
1243 global variable (optional).
1244
1245 ``check_for_duplicates``
1246 whether to check for duplicate entries when adding entries to the
1247 file (optional, default: ``False``).
1248 """
1249 enc = kwargs.get('encoding', default_encoding)
1250 if _is_file(pofile):
1251 try:
1252 self.fhandle = open(pofile, encoding=enc)
1253 except LookupError:
1254 enc = default_encoding
1255 self.fhandle = open(pofile, encoding=enc)
1256 else:
1257 self.fhandle = pofile.splitlines()
1258
1259 klass = kwargs.get('klass')
1260 if klass is None:
1261 klass = POFile
1262 self.instance = klass(
1263 pofile=pofile,
1264 encoding=enc,
1265 check_for_duplicates=kwargs.get('check_for_duplicates', False)
1266 )
1267 self.transitions = {}
1268 self.current_line = 0
1269 self.current_entry = POEntry(linenum=self.current_line)
1270 self.current_state = 'st'
1271 self.current_token = None
1272 # two memo flags used in handlers
1273 self.msgstr_index = 0
1274 self.entry_obsolete = 0
1275 # Configure the state machine, by adding transitions.
1276 # Signification of symbols:
1277 # * ST: Beginning of the file (start)
1278 # * HE: Header
1279 # * TC: a translation comment
1280 # * GC: a generated comment
1281 # * OC: a file/line occurrence
1282 # * FL: a flags line
1283 # * CT: a message context
1284 # * PC: a previous msgctxt
1285 # * PM: a previous msgid
1286 # * PP: a previous msgid_plural
1287 # * MI: a msgid
1288 # * MP: a msgid plural
1289 # * MS: a msgstr
1290 # * MX: a msgstr plural
1291 # * MC: a msgid or msgstr continuation line
1292 all = ['st', 'he', 'gc', 'oc', 'fl', 'ct', 'pc', 'pm', 'pp', 'tc',
1293 'ms', 'mp', 'mx', 'mi']
1294
1295 self.add('tc', ['st', 'he'], 'he')
1296 self.add('tc', ['gc', 'oc', 'fl', 'tc', 'pc', 'pm', 'pp', 'ms',
1297 'mp', 'mx', 'mi'], 'tc')
1298 self.add('gc', all, 'gc')
1299 self.add('oc', all, 'oc')
1300 self.add('fl', all, 'fl')
1301 self.add('pc', all, 'pc')
1302 self.add('pm', all, 'pm')
1303 self.add('pp', all, 'pp')
1304 self.add('ct', ['st', 'he', 'gc', 'oc', 'fl', 'tc', 'pc', 'pm',
1305 'pp', 'ms', 'mx'], 'ct')
1306 self.add('mi', ['st', 'he', 'gc', 'oc', 'fl', 'ct', 'tc', 'pc',
1307 'pm', 'pp', 'ms', 'mx'], 'mi')
1308 self.add('mp', ['tc', 'gc', 'pc', 'pm', 'pp', 'mi'], 'mp')
1309 self.add('ms', ['mi', 'mp', 'tc'], 'ms')
1310 self.add('mx', ['mi', 'mx', 'mp', 'tc'], 'mx')
1311 self.add('mc', ['ct', 'mi', 'mp', 'ms', 'mx', 'pm', 'pp', 'pc'], 'mc')
1312
1313 def parse(self):
1314 """
1315 Run the state machine, parse the file line by line and call process()
1316 with the current matched symbol.
1317 """
1318 try:
1319 keywords = {
1320 'msgctxt': 'ct',
1321 'msgid': 'mi',
1322 'msgstr': 'ms',
1323 'msgid_plural': 'mp',
1324 }
1325 prev_keywords = {
1326 'msgid_plural': 'pp',
1327 'msgid': 'pm',
1328 'msgctxt': 'pc',
1329 }
1330 tokens = []
1331 fpath = '%s ' % self.instance.fpath if self.instance.fpath else ''
1332 for line in self.fhandle:
1333 self.current_line += 1
1334 if self.current_line == 1:
1335 BOM = codecs.BOM_UTF8.decode('utf-8')
1336 if line.startswith(BOM):
1337 line = line[len(BOM):]
1338 line = line.strip()
1339 if line == '':
1340 continue
1341
1342 tokens = line.split(None, 2)
1343 nb_tokens = len(tokens)
1344
1345 if tokens[0] == '#~|':
1346 continue
1347
1348 if tokens[0] == '#~' and nb_tokens > 1:
1349 line = line[3:].strip()
1350 tokens = tokens[1:]
1351 nb_tokens -= 1
1352 self.entry_obsolete = 1
1353 else:
1354 self.entry_obsolete = 0
1355
1356 # Take care of keywords like
1357 # msgid, msgid_plural, msgctxt & msgstr.
1358 if tokens[0] in keywords and nb_tokens > 1:
1359 line = line[len(tokens[0]):].lstrip()
1360 if re.search(r'([^\\]|^)"', line[1:-1]):
1361 raise OSError('Syntax error in po file %s(line %s): '
1362 'unescaped double quote found' %
1363 (fpath, self.current_line))
1364 self.current_token = line
1365 self.process(keywords[tokens[0]])
1366 continue
1367
1368 self.current_token = line
1369
1370 if tokens[0] == '#:':
1371 if nb_tokens <= 1:
1372 continue
1373 # we are on a occurrences line
1374 self.process('oc')
1375
1376 elif line[:1] == '"':
1377 # we are on a continuation line
1378 if re.search(r'([^\\]|^)"', line[1:-1]):
1379 raise OSError('Syntax error in po file %s(line %s): '
1380 'unescaped double quote found' %
1381 (fpath, self.current_line))
1382 self.process('mc')
1383
1384 elif line[:7] == 'msgstr[':
1385 # we are on a msgstr plural
1386 self.process('mx')
1387
1388 elif tokens[0] == '#,':
1389 if nb_tokens <= 1:
1390 continue
1391 # we are on a flags line
1392 self.process('fl')
1393
1394 elif tokens[0] == '#' or tokens[0].startswith('##'):
1395 if line == '#':
1396 line += ' '
1397 # we are on a translator comment line
1398 self.process('tc')
1399
1400 elif tokens[0] == '#.':
1401 if nb_tokens <= 1:
1402 continue
1403 # we are on a generated comment line
1404 self.process('gc')
1405
1406 elif tokens[0] == '#|':
1407 if nb_tokens <= 1:
1408 raise OSError('Syntax error in po file %s(line %s)' %
1409 (fpath, self.current_line))
1410
1411 # Remove the marker and any whitespace right after that.
1412 line = line[2:].lstrip()
1413 self.current_token = line
1414
1415 if tokens[1].startswith('"'):
1416 # Continuation of previous metadata.
1417 self.process('mc')
1418 continue
1419
1420 if nb_tokens == 2:
1421 # Invalid continuation line.
1422 raise OSError('Syntax error in po file %s(line %s): '
1423 'invalid continuation line' %
1424 (fpath, self.current_line))
1425
1426 # we are on a "previous translation" comment line,
1427 if tokens[1] not in prev_keywords:
1428 # Unknown keyword in previous translation comment.
1429 raise OSError('Syntax error in po file %s(line %s): '
1430 'unknown keyword %s' %
1431 (fpath, self.current_line,
1432 tokens[1]))
1433
1434 # Remove the keyword and any whitespace
1435 # between it and the starting quote.
1436 line = line[len(tokens[1]):].lstrip()
1437 self.current_token = line
1438 self.process(prev_keywords[tokens[1]])
1439
1440 else:
1441 raise OSError('Syntax error in po file %s(line %s)' %
1442 (fpath, self.current_line))
1443
1444 if self.current_entry and len(tokens) > 0 and \
1445 not tokens[0].startswith('#'):
1446 # since entries are added when another entry is found, we must
1447 # add the last entry here (only if there are lines). Trailing
1448 # comments are ignored
1449 self.instance.append(self.current_entry)
1450
1451 # before returning the instance, check if there's metadata and if
1452 # so extract it in a dict
1453 metadataentry = self.instance.find('')
1454 if metadataentry: # metadata found
1455 # remove the entry
1456 self.instance.remove(metadataentry)
1457 self.instance.metadata_is_fuzzy = metadataentry.flags
1458 key = None
1459 for msg in metadataentry.msgstr.splitlines():
1460 try:
1461 key, val = msg.split(':', 1)
1462 self.instance.metadata[key] = val.strip()
1463 except (ValueError, KeyError):
1464 if key is not None:
1465 self.instance.metadata[key] += '\n' + msg.strip()
1466 finally:
1467 # close opened file
1468 if not isinstance(self.fhandle, list): # must be file
1469 self.fhandle.close()
1470 return self.instance
1471
1472 def add(self, symbol, states, next_state):
1473 """
1474 Add a transition to the state machine.
1475
1476 Keywords arguments:
1477
1478 ``symbol``
1479 string, the matched token (two chars symbol).
1480
1481 ``states``
1482 list, a list of states (two chars symbols).
1483
1484 ``next_state``
1485 the next state the fsm will have after the action.
1486 """
1487 for state in states:
1488 action = getattr(self, 'handle_%s' % next_state)
1489 self.transitions[(symbol, state)] = (action, next_state)
1490
1491 def process(self, symbol):
1492 """
1493 Process the transition corresponding to the current state and the
1494 symbol provided.
1495
1496 Keywords arguments:
1497
1498 ``symbol``
1499 string, the matched token (two chars symbol).
1500
1501 ``linenum``
1502 integer, the current line number of the parsed file.
1503 """
1504 try:
1505 (action, state) = self.transitions[(symbol, self.current_state)]
1506 if action():
1507 self.current_state = state
1508 except Exception:
1509 fpath = '%s ' % self.instance.fpath if self.instance.fpath else ''
1510 if hasattr(self.fhandle, 'close'):
1511 self.fhandle.close()
1512 raise OSError('Syntax error in po file %s(line %s)' %
1513 (fpath, self.current_line))
1514
1515 # state handlers
1516
1517 def handle_he(self):
1518 """Handle a header comment."""
1519 if self.instance.header != '':
1520 self.instance.header += '\n'
1521 self.instance.header += self.current_token[2:]
1522 return 1
1523
1524 def handle_tc(self):
1525 """Handle a translator comment."""
1526 if self.current_state in ['mc', 'ms', 'mx']:
1527 self.instance.append(self.current_entry)
1528 self.current_entry = POEntry(linenum=self.current_line)
1529 if self.current_entry.tcomment != '':
1530 self.current_entry.tcomment += '\n'
1531 tcomment = self.current_token.lstrip('#')
1532 if tcomment.startswith(' '):
1533 tcomment = tcomment[1:]
1534 self.current_entry.tcomment += tcomment
1535 return True
1536
1537 def handle_gc(self):
1538 """Handle a generated comment."""
1539 if self.current_state in ['mc', 'ms', 'mx']:
1540 self.instance.append(self.current_entry)
1541 self.current_entry = POEntry(linenum=self.current_line)
1542 if self.current_entry.comment != '':
1543 self.current_entry.comment += '\n'
1544 self.current_entry.comment += self.current_token[3:]
1545 return True
1546
1547 def handle_oc(self):
1548 """Handle a file:num occurrence."""
1549 if self.current_state in ['mc', 'ms', 'mx']:
1550 self.instance.append(self.current_entry)
1551 self.current_entry = POEntry(linenum=self.current_line)
1552 occurrences = self.current_token[3:].split()
1553 for occurrence in occurrences:
1554 if occurrence != '':
1555 try:
1556 fil, line = occurrence.rsplit(':', 1)
1557 if not line.isdigit():
1558 fil = occurrence
1559 line = ''
1560 self.current_entry.occurrences.append((fil, line))
1561 except (ValueError, AttributeError):
1562 self.current_entry.occurrences.append((occurrence, ''))
1563 return True
1564
1565 def handle_fl(self):
1566 """Handle a flags line."""
1567 if self.current_state in ['mc', 'ms', 'mx']:
1568 self.instance.append(self.current_entry)
1569 self.current_entry = POEntry(linenum=self.current_line)
1570 self.current_entry.flags += [c.strip() for c in
1571 self.current_token[3:].split(',')]
1572 return True
1573
1574 def handle_pp(self):
1575 """Handle a previous msgid_plural line."""
1576 if self.current_state in ['mc', 'ms', 'mx']:
1577 self.instance.append(self.current_entry)
1578 self.current_entry = POEntry(linenum=self.current_line)
1579 self.current_entry.previous_msgid_plural = \
1580 unescape(self.current_token[1:-1])
1581 return True
1582
1583 def handle_pm(self):
1584 """Handle a previous msgid line."""
1585 if self.current_state in ['mc', 'ms', 'mx']:
1586 self.instance.append(self.current_entry)
1587 self.current_entry = POEntry(linenum=self.current_line)
1588 self.current_entry.previous_msgid = \
1589 unescape(self.current_token[1:-1])
1590 return True
1591
1592 def handle_pc(self):
1593 """Handle a previous msgctxt line."""
1594 if self.current_state in ['mc', 'ms', 'mx']:
1595 self.instance.append(self.current_entry)
1596 self.current_entry = POEntry(linenum=self.current_line)
1597 self.current_entry.previous_msgctxt = \
1598 unescape(self.current_token[1:-1])
1599 return True
1600
1601 def handle_ct(self):
1602 """Handle a msgctxt."""
1603 if self.current_state in ['mc', 'ms', 'mx']:
1604 self.instance.append(self.current_entry)
1605 self.current_entry = POEntry(linenum=self.current_line)
1606 self.current_entry.msgctxt = unescape(self.current_token[1:-1])
1607 return True
1608
1609 def handle_mi(self):
1610 """Handle a msgid."""
1611 if self.current_state in ['mc', 'ms', 'mx']:
1612 self.instance.append(self.current_entry)
1613 self.current_entry = POEntry(linenum=self.current_line)
1614 self.current_entry.obsolete = self.entry_obsolete
1615 self.current_entry.msgid = unescape(self.current_token[1:-1])
1616 return True
1617
1618 def handle_mp(self):
1619 """Handle a msgid plural."""
1620 self.current_entry.msgid_plural = unescape(self.current_token[1:-1])
1621 return True
1622
1623 def handle_ms(self):
1624 """Handle a msgstr."""
1625 self.current_entry.msgstr = unescape(self.current_token[1:-1])
1626 return True
1627
1628 def handle_mx(self):
1629 """Handle a msgstr plural."""
1630 index = self.current_token[7]
1631 value = self.current_token[self.current_token.find('"') + 1:-1]
1632 self.current_entry.msgstr_plural[int(index)] = unescape(value)
1633 self.msgstr_index = int(index)
1634 return True
1635
1636 def handle_mc(self):
1637 """Handle a msgid or msgstr continuation line."""
1638 token = unescape(self.current_token[1:-1])
1639 if self.current_state == 'ct':
1640 self.current_entry.msgctxt += token
1641 elif self.current_state == 'mi':
1642 self.current_entry.msgid += token
1643 elif self.current_state == 'mp':
1644 self.current_entry.msgid_plural += token
1645 elif self.current_state == 'ms':
1646 self.current_entry.msgstr += token
1647 elif self.current_state == 'mx':
1648 self.current_entry.msgstr_plural[self.msgstr_index] += token
1649 elif self.current_state == 'pp':
1650 self.current_entry.previous_msgid_plural += token
1651 elif self.current_state == 'pm':
1652 self.current_entry.previous_msgid += token
1653 elif self.current_state == 'pc':
1654 self.current_entry.previous_msgctxt += token
1655 # don't change the current state
1656 return False
1657 # }}}
1658
1659
1660 # class _MOFileParser {{{
1661 class _MOFileParser:
1662 """
1663 A class to parse binary mo files.
1664 """
1665
1666 def __init__(self, mofile, *args, **kwargs):
1667 """
1668 Constructor.
1669
1670 Keyword arguments:
1671
1672 ``mofile``
1673 string, path to the mo file or its content
1674
1675 ``encoding``
1676 string, the encoding to use, defaults to ``default_encoding``
1677 global variable (optional).
1678
1679 ``check_for_duplicates``
1680 whether to check for duplicate entries when adding entries to the
1681 file (optional, default: ``False``).
1682 """
1683 if _is_file(mofile):
1684 self.fhandle = open(mofile, 'rb')
1685 else:
1686 self.fhandle = io.BytesIO(mofile)
1687
1688 klass = kwargs.get('klass')
1689 if klass is None:
1690 klass = MOFile
1691 self.instance = klass(
1692 fpath=mofile,
1693 encoding=kwargs.get('encoding', default_encoding),
1694 check_for_duplicates=kwargs.get('check_for_duplicates', False)
1695 )
1696
1697 def __del__(self):
1698 """
1699 Make sure the file is closed, this prevents warnings on unclosed file
1700 when running tests with python >= 3.2.
1701 """
1702 if self.fhandle and hasattr(self.fhandle, 'close'):
1703 self.fhandle.close()
1704
1705 def parse(self):
1706 """
1707 Build the instance with the file handle provided in the
1708 constructor.
1709 """
1710 # parse magic number
1711 magic_number = self._readbinary('<I', 4)
1712 if magic_number == MOFile.MAGIC:
1713 ii = '<II'
1714 elif magic_number == MOFile.MAGIC_SWAPPED:
1715 ii = '>II'
1716 else:
1717 raise OSError('Invalid mo file, magic number is incorrect !')
1718 self.instance.magic_number = magic_number
1719 # parse the version number and the number of strings
1720 version, numofstrings = self._readbinary(ii, 8)
1721 # from MO file format specs: "A program seeing an unexpected major
1722 # revision number should stop reading the MO file entirely"
1723 if version >> 16 not in (0, 1):
1724 raise OSError('Invalid mo file, unexpected major revision number')
1725 self.instance.version = version
1726 # original strings and translation strings hash table offset
1727 msgids_hash_offset, msgstrs_hash_offset = self._readbinary(ii, 8)
1728 # move to msgid hash table and read length and offset of msgids
1729 self.fhandle.seek(msgids_hash_offset)
1730 msgids_index = []
1731 for i in range(numofstrings):
1732 msgids_index.append(self._readbinary(ii, 8))
1733 # move to msgstr hash table and read length and offset of msgstrs
1734 self.fhandle.seek(msgstrs_hash_offset)
1735 msgstrs_index = []
1736 for i in range(numofstrings):
1737 msgstrs_index.append(self._readbinary(ii, 8))
1738 # build entries
1739 encoding = self.instance.encoding
1740 for i in range(numofstrings):
1741 self.fhandle.seek(msgids_index[i][1])
1742 msgid = self.fhandle.read(msgids_index[i][0])
1743
1744 self.fhandle.seek(msgstrs_index[i][1])
1745 msgstr = self.fhandle.read(msgstrs_index[i][0])
1746 if i == 0 and not msgid: # metadata
1747 raw_metadata, metadata = msgstr.split(b('\n')), {}
1748 for line in raw_metadata:
1749 tokens = line.split(b(':'), 1)
1750 if tokens[0] != b(''):
1751 try:
1752 k = tokens[0].decode(encoding)
1753 v = tokens[1].decode(encoding)
1754 metadata[k] = v.strip()
1755 except IndexError:
1756 metadata[k] = u('')
1757 self.instance.metadata = metadata
1758 continue
1759 # test if we have a plural entry
1760 msgid_tokens = msgid.split(b('\0'))
1761 if len(msgid_tokens) > 1:
1762 entry = self._build_entry(
1763 msgid=msgid_tokens[0],
1764 msgid_plural=msgid_tokens[1],
1765 msgstr_plural={k: v for k, v in
1766 enumerate(msgstr.split(b('\0')))}
1767 )
1768 else:
1769 entry = self._build_entry(msgid=msgid, msgstr=msgstr)
1770 self.instance.append(entry)
1771 # close opened file
1772 self.fhandle.close()
1773 return self.instance
1774
1775 def _build_entry(self, msgid, msgstr=None, msgid_plural=None,
1776 msgstr_plural=None):
1777 msgctxt_msgid = msgid.split(b('\x04'))
1778 encoding = self.instance.encoding
1779 if len(msgctxt_msgid) > 1:
1780 kwargs = {
1781 'msgctxt': msgctxt_msgid[0].decode(encoding),
1782 'msgid': msgctxt_msgid[1].decode(encoding),
1783 }
1784 else:
1785 kwargs = {'msgid': msgid.decode(encoding)}
1786 if msgstr:
1787 kwargs['msgstr'] = msgstr.decode(encoding)
1788 if msgid_plural:
1789 kwargs['msgid_plural'] = msgid_plural.decode(encoding)
1790 if msgstr_plural:
1791 for k in msgstr_plural:
1792 msgstr_plural[k] = msgstr_plural[k].decode(encoding)
1793 kwargs['msgstr_plural'] = msgstr_plural
1794 return MOEntry(**kwargs)
1795
1796 def _readbinary(self, fmt, numbytes):
1797 """
1798 Private method that unpack n bytes of data using format <fmt>.
1799 It returns a tuple or a mixed value if the tuple length is 1.
1800 """
1801 bytes = self.fhandle.read(numbytes)
1802 tup = struct.unpack(fmt, bytes)
1803 if len(tup) == 1:
1804 return tup[0]
1805 return tup
1806 # }}}
1807
1808 if __name__ == '__main__':
1809 pass