Debian Codegraph @

Lib/test/test_tools/test_msgfmt.py

raw
1 """Tests for the Tools/i18n/msgfmt.py tool."""
2
3 import json
4 import struct
5 import sys
6 import unittest
7 from gettext import GNUTranslations
8 from pathlib import Path
9
10 from test.support.os_helper import temp_cwd
11 from test.support.script_helper import assert_python_failure, assert_python_ok
12 from test.test_tools import imports_under_tool, skip_if_missing, toolsdir
13
14
15 skip_if_missing('i18n')
16
17 data_dir = (Path(__file__).parent / 'msgfmt_data').resolve()
18 script_dir = Path(toolsdir) / 'i18n'
19 msgfmt_py = script_dir / 'msgfmt.py'
20
21 with imports_under_tool("i18n"):
22 import msgfmt
23
24
25 def compile_messages(po_file, mo_file):
26 assert_python_ok(msgfmt_py, '-o', mo_file, po_file)
27
28
29 class CompilationTest(unittest.TestCase):
30
31 def test_compilation(self):
32 self.maxDiff = None
33 with temp_cwd():
34 for po_file in data_dir.glob('*.po'):
35 with self.subTest(po_file=po_file):
36 mo_file = po_file.with_suffix('.mo')
37 with open(mo_file, 'rb') as f:
38 expected = GNUTranslations(f)
39
40 tmp_mo_file = mo_file.name
41 compile_messages(po_file, tmp_mo_file)
42 with open(tmp_mo_file, 'rb') as f:
43 actual = GNUTranslations(f)
44
45 self.assertDictEqual(actual._catalog, expected._catalog)
46
47 def test_binary_header(self):
48 with temp_cwd():
49 tmp_mo_file = 'messages.mo'
50 compile_messages(data_dir / "general.po", tmp_mo_file)
51 with open(tmp_mo_file, 'rb') as f:
52 mo_data = f.read()
53
54 (
55 magic,
56 version,
57 num_strings,
58 orig_table_offset,
59 trans_table_offset,
60 hash_table_size,
61 hash_table_offset,
62 ) = struct.unpack("=7I", mo_data[:28])
63
64 self.assertEqual(magic, 0x950412de)
65 self.assertEqual(version, 0)
66 self.assertEqual(num_strings, 9)
67 self.assertEqual(orig_table_offset, 28)
68 self.assertEqual(trans_table_offset, 100)
69 self.assertEqual(hash_table_size, 0)
70 self.assertEqual(hash_table_offset, 0)
71
72 def test_translations(self):
73 with open(data_dir / 'general.mo', 'rb') as f:
74 t = GNUTranslations(f)
75
76 self.assertEqual(t.gettext('foo'), 'foo')
77 self.assertEqual(t.gettext('bar'), 'baz')
78 self.assertEqual(t.pgettext('abc', 'foo'), 'bar')
79 self.assertEqual(t.pgettext('xyz', 'foo'), 'bar')
80 self.assertEqual(t.gettext('Multilinestring'), 'Multilinetranslation')
81 self.assertEqual(t.gettext('"escapes"'), '"translated"')
82 self.assertEqual(t.gettext('\n newlines \n'), '\n translated \n')
83 self.assertEqual(t.ngettext('One email sent.', '%d emails sent.', 1),
84 'One email sent.')
85 self.assertEqual(t.ngettext('One email sent.', '%d emails sent.', 2),
86 '%d emails sent.')
87 self.assertEqual(t.npgettext('abc', 'One email sent.',
88 '%d emails sent.', 1),
89 'One email sent.')
90 self.assertEqual(t.npgettext('abc', 'One email sent.',
91 '%d emails sent.', 2),
92 '%d emails sent.')
93
94 def test_po_with_bom(self):
95 with temp_cwd():
96 Path('bom.po').write_bytes(b'\xef\xbb\xbfmsgid "Python"\nmsgstr "Pioton"\n')
97
98 res = assert_python_failure(msgfmt_py, 'bom.po')
99 err = res.err.decode('utf-8')
100 self.assertIn('The file bom.po starts with a UTF-8 BOM', err)
101
102 def test_invalid_msgid_plural(self):
103 with temp_cwd():
104 Path('invalid.po').write_text('''\
105 msgid_plural "plural"
106 msgstr[0] "singular"
107 ''')
108
109 res = assert_python_failure(msgfmt_py, 'invalid.po')
110 err = res.err.decode('utf-8')
111 self.assertIn('msgid_plural not preceded by msgid', err)
112
113 def test_plural_without_msgid_plural(self):
114 with temp_cwd():
115 Path('invalid.po').write_text('''\
116 msgid "foo"
117 msgstr[0] "bar"
118 ''')
119
120 res = assert_python_failure(msgfmt_py, 'invalid.po')
121 err = res.err.decode('utf-8')
122 self.assertIn('plural without msgid_plural', err)
123
124 def test_indexed_msgstr_without_msgid_plural(self):
125 with temp_cwd():
126 Path('invalid.po').write_text('''\
127 msgid "foo"
128 msgid_plural "foos"
129 msgstr "bar"
130 ''')
131
132 res = assert_python_failure(msgfmt_py, 'invalid.po')
133 err = res.err.decode('utf-8')
134 self.assertIn('indexed msgstr required for plural', err)
135
136 def test_generic_syntax_error(self):
137 with temp_cwd():
138 Path('invalid.po').write_text('''\
139 "foo"
140 ''')
141
142 res = assert_python_failure(msgfmt_py, 'invalid.po')
143 err = res.err.decode('utf-8')
144 self.assertIn('Syntax error', err)
145
146
147 class POParserTest(unittest.TestCase):
148 @classmethod
149 def tearDownClass(cls):
150 # msgfmt uses a global variable to store messages,
151 # clear it after the tests.
152 msgfmt.MESSAGES.clear()
153
154 def test_strings(self):
155 # Test that the PO parser correctly handles and unescape
156 # strings in the PO file.
157 # The PO file format allows for a variety of escape sequences,
158 # octal and hex escapes.
159 valid_strings = (
160 # empty strings
161 ('""', ''),
162 ('"" "" ""', ''),
163 # allowed escape sequences
164 (r'"\\"', '\\'),
165 (r'"\""', '"'),
166 (r'"\t"', '\t'),
167 (r'"\n"', '\n'),
168 (r'"\r"', '\r'),
169 (r'"\f"', '\f'),
170 (r'"\a"', '\a'),
171 (r'"\b"', '\b'),
172 (r'"\v"', '\v'),
173 # non-empty strings
174 ('"foo"', 'foo'),
175 ('"foo" "bar"', 'foobar'),
176 ('"foo""bar"', 'foobar'),
177 ('"" "foo" ""', 'foo'),
178 # newlines and tabs
179 (r'"foo\nbar"', 'foo\nbar'),
180 (r'"foo\n" "bar"', 'foo\nbar'),
181 (r'"foo\tbar"', 'foo\tbar'),
182 (r'"foo\t" "bar"', 'foo\tbar'),
183 # escaped quotes
184 (r'"foo\"bar"', 'foo"bar'),
185 (r'"foo\"" "bar"', 'foo"bar'),
186 (r'"foo\\" "bar"', 'foo\\bar'),
187 # octal escapes
188 (r'"\120\171\164\150\157\156"', 'Python'),
189 (r'"\120\171\164" "\150\157\156"', 'Python'),
190 (r'"\"\120\171\164" "\150\157\156\""', '"Python"'),
191 # hex escapes
192 (r'"\x50\x79\x74\x68\x6f\x6e"', 'Python'),
193 (r'"\x50\x79\x74" "\x68\x6f\x6e"', 'Python'),
194 (r'"\"\x50\x79\x74" "\x68\x6f\x6e\""', '"Python"'),
195 )
196
197 with temp_cwd():
198 for po_string, expected in valid_strings:
199 with self.subTest(po_string=po_string):
200 # Construct a PO file with a single entry,
201 # compile it, read it into a catalog and
202 # check the result.
203 po = f'msgid {po_string}\nmsgstr "translation"'
204 Path('messages.po').write_text(po)
205 # Reset the global MESSAGES dictionary
206 msgfmt.MESSAGES.clear()
207 msgfmt.make('messages.po', 'messages.mo')
208
209 with open('messages.mo', 'rb') as f:
210 actual = GNUTranslations(f)
211
212 self.assertDictEqual(actual._catalog, {expected: 'translation'})
213
214 invalid_strings = (
215 # "''", # invalid but currently accepted
216 '"',
217 '"""',
218 '"" "',
219 'foo',
220 '"" "foo',
221 '"foo" foo',
222 '42',
223 '"" 42 ""',
224 # disallowed escape sequences
225 # r'"\'"', # invalid but currently accepted
226 # r'"\e"', # invalid but currently accepted
227 # r'"\8"', # invalid but currently accepted
228 # r'"\9"', # invalid but currently accepted
229 r'"\x"',
230 r'"\u1234"',
231 r'"\N{ROMAN NUMERAL NINE}"'
232 )
233 with temp_cwd():
234 for invalid_string in invalid_strings:
235 with self.subTest(string=invalid_string):
236 po = f'msgid {invalid_string}\nmsgstr "translation"'
237 Path('messages.po').write_text(po)
238 # Reset the global MESSAGES dictionary
239 msgfmt.MESSAGES.clear()
240 with self.assertRaises(Exception):
241 msgfmt.make('messages.po', 'messages.mo')
242
243
244 class CLITest(unittest.TestCase):
245
246 def test_help(self):
247 for option in ('--help', '-h'):
248 res = assert_python_ok(msgfmt_py, option)
249 err = res.err.decode('utf-8')
250 self.assertIn('Generate binary message catalog from textual translation description.', err)
251
252 def test_version(self):
253 for option in ('--version', '-V'):
254 res = assert_python_ok(msgfmt_py, option)
255 out = res.out.decode('utf-8').strip()
256 self.assertEqual('msgfmt.py 1.2', out)
257
258 def test_invalid_option(self):
259 res = assert_python_failure(msgfmt_py, '--invalid-option')
260 err = res.err.decode('utf-8')
261 self.assertIn('Generate binary message catalog from textual translation description.', err)
262 self.assertIn('option --invalid-option not recognized', err)
263
264 def test_no_input_file(self):
265 res = assert_python_ok(msgfmt_py)
266 err = res.err.decode('utf-8').replace('\r\n', '\n')
267 self.assertIn('No input file given\n'
268 "Try `msgfmt --help' for more information.", err)
269
270 def test_nonexistent_file(self):
271 assert_python_failure(msgfmt_py, 'nonexistent.po')
272
273
274 def update_catalog_snapshots():
275 for po_file in data_dir.glob('*.po'):
276 mo_file = po_file.with_suffix('.mo')
277 compile_messages(po_file, mo_file)
278 # Create a human-readable JSON file which is
279 # easier to review than the binary .mo file.
280 with open(mo_file, 'rb') as f:
281 translations = GNUTranslations(f)
282 catalog_file = po_file.with_suffix('.json')
283 with open(catalog_file, 'w') as f:
284 data = translations._catalog.items()
285 data = sorted(data, key=lambda x: (isinstance(x[0], tuple), x[0]))
286 json.dump(data, f, indent=4)
287 f.write('\n')
288
289
290 if __name__ == '__main__':
291 if len(sys.argv) > 1 and sys.argv[1] == '--snapshot-update':
292 update_catalog_snapshots()
293 sys.exit(0)
294 unittest.main()