Debian Codegraph @

tests/i18n/test_extraction.py

raw
1 import os
2 import re
3 import shutil
4 import tempfile
5 import time
6 import warnings
7 from io import StringIO
8 from pathlib import Path
9 from unittest import mock, skipUnless
10
11 from admin_scripts.tests import AdminScriptTestCase
12
13 from django.core import management
14 from django.core.management import execute_from_command_line
15 from django.core.management.base import CommandError
16 from django.core.management.commands.makemessages import Command as MakeMessagesCommand
17 from django.core.management.commands.makemessages import write_pot_file
18 from django.core.management.utils import find_command
19 from django.test import SimpleTestCase, override_settings
20 from django.test.utils import captured_stderr, captured_stdout
21 from django.utils._os import symlinks_supported
22 from django.utils.translation import TranslatorCommentWarning
23
24 from .utils import POFileAssertionMixin, RunInTmpDirMixin, copytree
25
26 LOCALE = "de"
27 has_xgettext = find_command("xgettext")
28
29
30 @skipUnless(has_xgettext, "xgettext is mandatory for extraction tests")
31 class ExtractorTests(POFileAssertionMixin, RunInTmpDirMixin, SimpleTestCase):
32 work_subdir = "commands"
33
34 PO_FILE = "locale/%s/LC_MESSAGES/django.po" % LOCALE
35
36 def _run_makemessages(self, **options):
37 out = StringIO()
38 management.call_command(
39 "makemessages", locale=[LOCALE], verbosity=2, stdout=out, **options
40 )
41 output = out.getvalue()
42 self.assertTrue(os.path.exists(self.PO_FILE))
43 with open(self.PO_FILE) as fp:
44 po_contents = fp.read()
45 return output, po_contents
46
47 def assertMsgIdPlural(self, msgid, haystack, use_quotes=True):
48 return self._assertPoKeyword(
49 "msgid_plural", msgid, haystack, use_quotes=use_quotes
50 )
51
52 def assertMsgStr(self, msgstr, haystack, use_quotes=True):
53 return self._assertPoKeyword("msgstr", msgstr, haystack, use_quotes=use_quotes)
54
55 def assertNotMsgId(self, msgid, s, use_quotes=True):
56 if use_quotes:
57 msgid = '"%s"' % msgid
58 msgid = re.escape(msgid)
59 return self.assertTrue(not re.search("^msgid %s" % msgid, s, re.MULTILINE))
60
61 def _assertPoLocComment(
62 self, assert_presence, po_filename, line_number, *comment_parts
63 ):
64 with open(po_filename) as fp:
65 po_contents = fp.read()
66 if os.name == "nt":
67 # #: .\path\to\file.html:123
68 cwd_prefix = "%s%s" % (os.curdir, os.sep)
69 else:
70 # #: path/to/file.html:123
71 cwd_prefix = ""
72
73 path = os.path.join(cwd_prefix, *comment_parts)
74 parts = [path]
75
76 if isinstance(line_number, str):
77 line_number = self._get_token_line_number(path, line_number)
78 if line_number is not None:
79 parts.append(":%d" % line_number)
80
81 needle = "".join(parts)
82 pattern = re.compile(r"^\#\:.*" + re.escape(needle), re.MULTILINE)
83 if assert_presence:
84 return self.assertRegex(
85 po_contents, pattern, '"%s" not found in final .po file.' % needle
86 )
87 else:
88 return self.assertNotRegex(
89 po_contents, pattern, '"%s" shouldn\'t be in final .po file.' % needle
90 )
91
92 def _get_token_line_number(self, path, token):
93 with open(path) as f:
94 for line, content in enumerate(f, 1):
95 if token in content:
96 return line
97 self.fail(
98 "The token '%s' could not be found in %s, please check the test config"
99 % (token, path)
100 )
101
102 def assertLocationCommentPresent(self, po_filename, line_number, *comment_parts):
103 r"""
104 self.assertLocationCommentPresent('django.po', 42, 'dirA', 'dirB', 'foo.py')
105
106 verifies that the django.po file has a gettext-style location comment
107 of the form
108
109 `#: dirA/dirB/foo.py:42`
110
111 (or `#: .\dirA\dirB\foo.py:42` on Windows)
112
113 None can be passed for the line_number argument to skip checking of
114 the :42 suffix part.
115 A string token can also be passed as line_number, in which case it
116 will be searched in the template, and its line number will be used.
117 A msgid is a suitable candidate.
118 """
119 return self._assertPoLocComment(True, po_filename, line_number, *comment_parts)
120
121 def assertLocationCommentNotPresent(self, po_filename, line_number, *comment_parts):
122 """Check the opposite of assertLocationComment()"""
123 return self._assertPoLocComment(False, po_filename, line_number, *comment_parts)
124
125 def assertRecentlyModified(self, path):
126 """
127 Assert that file was recently modified (modification time was less than
128 10 seconds ago).
129 """
130 delta = time.time() - os.stat(path).st_mtime
131 self.assertLess(delta, 10, "%s was recently modified" % path)
132
133 def assertNotRecentlyModified(self, path):
134 """
135 Assert that file was not recently modified (modification time was more
136 than 10 seconds ago).
137 """
138 delta = time.time() - os.stat(path).st_mtime
139 self.assertGreater(delta, 10, "%s wasn't recently modified" % path)
140
141
142 class BasicExtractorTests(ExtractorTests):
143 @override_settings(USE_I18N=False)
144 def test_use_i18n_false(self):
145 """
146 makemessages also runs successfully when USE_I18N is False.
147 """
148 management.call_command("makemessages", locale=[LOCALE], verbosity=0)
149 self.assertTrue(os.path.exists(self.PO_FILE))
150 with open(self.PO_FILE, encoding="utf-8") as fp:
151 po_contents = fp.read()
152 # Check two random strings
153 self.assertIn("#. Translators: One-line translator comment #1", po_contents)
154 self.assertIn('msgctxt "Special trans context #1"', po_contents)
155
156 def test_no_option(self):
157 # One of either the --locale, --exclude, or --all options is required.
158 msg = "Type 'manage.py help makemessages' for usage information."
159 with mock.patch(
160 "django.core.management.commands.makemessages.sys.argv",
161 ["manage.py", "makemessages"],
162 ):
163 with self.assertRaisesRegex(CommandError, msg):
164 management.call_command("makemessages")
165
166 def test_valid_locale(self):
167 out = StringIO()
168 management.call_command("makemessages", locale=["de"], stdout=out, verbosity=1)
169 self.assertNotIn("invalid locale de", out.getvalue())
170 self.assertIn("processing locale de", out.getvalue())
171 self.assertIs(Path(self.PO_FILE).exists(), True)
172
173 def test_valid_locale_with_country(self):
174 out = StringIO()
175 management.call_command(
176 "makemessages", locale=["en_GB"], stdout=out, verbosity=1
177 )
178 self.assertNotIn("invalid locale en_GB", out.getvalue())
179 self.assertIn("processing locale en_GB", out.getvalue())
180 self.assertIs(Path("locale/en_GB/LC_MESSAGES/django.po").exists(), True)
181
182 def test_valid_locale_with_numeric_region_code(self):
183 out = StringIO()
184 management.call_command(
185 "makemessages", locale=["ar_002"], stdout=out, verbosity=1
186 )
187 self.assertNotIn("invalid locale ar_002", out.getvalue())
188 self.assertIn("processing locale ar_002", out.getvalue())
189 self.assertIs(Path("locale/ar_002/LC_MESSAGES/django.po").exists(), True)
190
191 def test_valid_locale_tachelhit_latin_morocco(self):
192 out = StringIO()
193 management.call_command(
194 "makemessages", locale=["shi_Latn_MA"], stdout=out, verbosity=1
195 )
196 self.assertNotIn("invalid locale shi_Latn_MA", out.getvalue())
197 self.assertIn("processing locale shi_Latn_MA", out.getvalue())
198 self.assertIs(Path("locale/shi_Latn_MA/LC_MESSAGES/django.po").exists(), True)
199
200 def test_valid_locale_private_subtag(self):
201 out = StringIO()
202 management.call_command(
203 "makemessages", locale=["nl_NL-x-informal"], stdout=out, verbosity=1
204 )
205 self.assertNotIn("invalid locale nl_NL-x-informal", out.getvalue())
206 self.assertIn("processing locale nl_NL-x-informal", out.getvalue())
207 self.assertIs(
208 Path("locale/nl_NL-x-informal/LC_MESSAGES/django.po").exists(), True
209 )
210
211 def test_invalid_locale_uppercase(self):
212 out = StringIO()
213 management.call_command("makemessages", locale=["PL"], stdout=out, verbosity=1)
214 self.assertIn("invalid locale PL, did you mean pl?", out.getvalue())
215 self.assertNotIn("processing locale pl", out.getvalue())
216 self.assertIs(Path("locale/pl/LC_MESSAGES/django.po").exists(), False)
217
218 def test_invalid_locale_hyphen(self):
219 out = StringIO()
220 management.call_command(
221 "makemessages", locale=["pl-PL"], stdout=out, verbosity=1
222 )
223 self.assertIn("invalid locale pl-PL, did you mean pl_PL?", out.getvalue())
224 self.assertNotIn("processing locale pl-PL", out.getvalue())
225 self.assertIs(Path("locale/pl-PL/LC_MESSAGES/django.po").exists(), False)
226
227 def test_invalid_locale_lower_country(self):
228 out = StringIO()
229 management.call_command(
230 "makemessages", locale=["pl_pl"], stdout=out, verbosity=1
231 )
232 self.assertIn("invalid locale pl_pl, did you mean pl_PL?", out.getvalue())
233 self.assertNotIn("processing locale pl_pl", out.getvalue())
234 self.assertIs(Path("locale/pl_pl/LC_MESSAGES/django.po").exists(), False)
235
236 def test_invalid_locale_private_subtag(self):
237 out = StringIO()
238 management.call_command(
239 "makemessages", locale=["nl-nl-x-informal"], stdout=out, verbosity=1
240 )
241 self.assertIn(
242 "invalid locale nl-nl-x-informal, did you mean nl_NL-x-informal?",
243 out.getvalue(),
244 )
245 self.assertNotIn("processing locale nl-nl-x-informal", out.getvalue())
246 self.assertIs(
247 Path("locale/nl-nl-x-informal/LC_MESSAGES/django.po").exists(), False
248 )
249
250 def test_invalid_locale_plus(self):
251 out = StringIO()
252 management.call_command(
253 "makemessages", locale=["en+GB"], stdout=out, verbosity=1
254 )
255 self.assertIn("invalid locale en+GB, did you mean en_GB?", out.getvalue())
256 self.assertNotIn("processing locale en+GB", out.getvalue())
257 self.assertIs(Path("locale/en+GB/LC_MESSAGES/django.po").exists(), False)
258
259 def test_invalid_locale_end_with_underscore(self):
260 out = StringIO()
261 management.call_command("makemessages", locale=["en_"], stdout=out, verbosity=1)
262 self.assertIn("invalid locale en_", out.getvalue())
263 self.assertNotIn("processing locale en_", out.getvalue())
264 self.assertIs(Path("locale/en_/LC_MESSAGES/django.po").exists(), False)
265
266 def test_invalid_locale_start_with_underscore(self):
267 out = StringIO()
268 management.call_command("makemessages", locale=["_en"], stdout=out, verbosity=1)
269 self.assertIn("invalid locale _en", out.getvalue())
270 self.assertNotIn("processing locale _en", out.getvalue())
271 self.assertIs(Path("locale/_en/LC_MESSAGES/django.po").exists(), False)
272
273 def test_comments_extractor(self):
274 management.call_command("makemessages", locale=[LOCALE], verbosity=0)
275 self.assertTrue(os.path.exists(self.PO_FILE))
276 with open(self.PO_FILE, encoding="utf-8") as fp:
277 po_contents = fp.read()
278 self.assertNotIn("This comment should not be extracted", po_contents)
279
280 # Comments in templates
281 self.assertIn(
282 "#. Translators: This comment should be extracted", po_contents
283 )
284 self.assertIn(
285 "#. Translators: Django comment block for translators\n#. "
286 "string's meaning unveiled",
287 po_contents,
288 )
289 self.assertIn("#. Translators: One-line translator comment #1", po_contents)
290 self.assertIn(
291 "#. Translators: Two-line translator comment #1\n#. continued here.",
292 po_contents,
293 )
294 self.assertIn("#. Translators: One-line translator comment #2", po_contents)
295 self.assertIn(
296 "#. Translators: Two-line translator comment #2\n#. continued here.",
297 po_contents,
298 )
299 self.assertIn("#. Translators: One-line translator comment #3", po_contents)
300 self.assertIn(
301 "#. Translators: Two-line translator comment #3\n#. continued here.",
302 po_contents,
303 )
304 self.assertIn("#. Translators: One-line translator comment #4", po_contents)
305 self.assertIn(
306 "#. Translators: Two-line translator comment #4\n#. continued here.",
307 po_contents,
308 )
309 self.assertIn(
310 "#. Translators: One-line translator comment #5 -- with "
311 "non ASCII characters: áéíóúö",
312 po_contents,
313 )
314 self.assertIn(
315 "#. Translators: Two-line translator comment #5 -- with "
316 "non ASCII characters: áéíóúö\n#. continued here.",
317 po_contents,
318 )
319
320 def test_special_char_extracted(self):
321 management.call_command("makemessages", locale=[LOCALE], verbosity=0)
322 self.assertTrue(os.path.exists(self.PO_FILE))
323 with open(self.PO_FILE, encoding="utf-8") as fp:
324 po_contents = fp.read()
325 self.assertMsgId("Non-breaking space\u00a0:", po_contents)
326
327 def test_blocktranslate_trimmed(self):
328 management.call_command("makemessages", locale=[LOCALE], verbosity=0)
329 self.assertTrue(os.path.exists(self.PO_FILE))
330 with open(self.PO_FILE) as fp:
331 po_contents = fp.read()
332 # should not be trimmed
333 self.assertNotMsgId("Text with a few line breaks.", po_contents)
334 # should be trimmed
335 self.assertMsgId(
336 "Again some text with a few line breaks, this time should be trimmed.",
337 po_contents,
338 )
339 # #21406 -- Should adjust for eaten line numbers
340 self.assertMsgId("Get my line number", po_contents)
341 self.assertLocationCommentPresent(
342 self.PO_FILE, "Get my line number", "templates", "test.html"
343 )
344
345 def test_extraction_error(self):
346 msg = (
347 "Translation blocks must not include other block tags: blocktranslate "
348 "(file %s, line 3)" % os.path.join("templates", "template_with_error.tpl")
349 )
350 with self.assertRaisesMessage(SyntaxError, msg):
351 management.call_command(
352 "makemessages", locale=[LOCALE], extensions=["tpl"], verbosity=0
353 )
354 # The temporary files were cleaned up.
355 self.assertFalse(os.path.exists("./templates/template_with_error.tpl.py"))
356 self.assertFalse(os.path.exists("./templates/template_0_with_no_error.tpl.py"))
357
358 def test_unicode_decode_error(self):
359 shutil.copyfile("./not_utf8.sample", "./not_utf8.txt")
360 out = StringIO()
361 management.call_command("makemessages", locale=[LOCALE], stdout=out)
362 self.assertIn(
363 "UnicodeDecodeError: skipped file not_utf8.txt in .", out.getvalue()
364 )
365
366 def test_unicode_file_name(self):
367 open(os.path.join(self.test_dir, "vidéo.txt"), "a").close()
368 management.call_command("makemessages", locale=[LOCALE], verbosity=0)
369
370 def test_extraction_warning(self):
371 """test xgettext warning about multiple bare interpolation placeholders"""
372 shutil.copyfile("./code.sample", "./code_sample.py")
373 out = StringIO()
374 management.call_command("makemessages", locale=[LOCALE], stdout=out)
375 self.assertIn("code_sample.py:4", out.getvalue())
376
377 def test_template_message_context_extractor(self):
378 """
379 Message contexts are correctly extracted for the {% translate %} and
380 {% blocktranslate %} template tags (#14806).
381 """
382 management.call_command("makemessages", locale=[LOCALE], verbosity=0)
383 self.assertTrue(os.path.exists(self.PO_FILE))
384 with open(self.PO_FILE) as fp:
385 po_contents = fp.read()
386 # {% translate %}
387 self.assertIn('msgctxt "Special trans context #1"', po_contents)
388 self.assertMsgId("Translatable literal #7a", po_contents)
389 self.assertIn('msgctxt "Special trans context #2"', po_contents)
390 self.assertMsgId("Translatable literal #7b", po_contents)
391 self.assertIn('msgctxt "Special trans context #3"', po_contents)
392 self.assertMsgId("Translatable literal #7c", po_contents)
393
394 # {% translate %} with a filter
395 for (
396 minor_part
397 ) in "abcdefgh": # Iterate from #7.1a to #7.1h template markers
398 self.assertIn(
399 'msgctxt "context #7.1{}"'.format(minor_part), po_contents
400 )
401 self.assertMsgId(
402 "Translatable literal #7.1{}".format(minor_part), po_contents
403 )
404
405 # {% blocktranslate %}
406 self.assertIn('msgctxt "Special blocktranslate context #1"', po_contents)
407 self.assertMsgId("Translatable literal #8a", po_contents)
408 self.assertIn('msgctxt "Special blocktranslate context #2"', po_contents)
409 self.assertMsgId("Translatable literal #8b-singular", po_contents)
410 self.assertIn("Translatable literal #8b-plural", po_contents)
411 self.assertIn('msgctxt "Special blocktranslate context #3"', po_contents)
412 self.assertMsgId("Translatable literal #8c-singular", po_contents)
413 self.assertIn("Translatable literal #8c-plural", po_contents)
414 self.assertIn('msgctxt "Special blocktranslate context #4"', po_contents)
415 self.assertMsgId("Translatable literal #8d %(a)s", po_contents)
416
417 # {% trans %} and {% blocktrans %}
418 self.assertMsgId("trans text", po_contents)
419 self.assertMsgId("blocktrans text", po_contents)
420
421 def test_context_in_single_quotes(self):
422 management.call_command("makemessages", locale=[LOCALE], verbosity=0)
423 self.assertTrue(os.path.exists(self.PO_FILE))
424 with open(self.PO_FILE) as fp:
425 po_contents = fp.read()
426 # {% translate %}
427 self.assertIn('msgctxt "Context wrapped in double quotes"', po_contents)
428 self.assertIn('msgctxt "Context wrapped in single quotes"', po_contents)
429
430 # {% blocktranslate %}
431 self.assertIn(
432 'msgctxt "Special blocktranslate context wrapped in double quotes"',
433 po_contents,
434 )
435 self.assertIn(
436 'msgctxt "Special blocktranslate context wrapped in single quotes"',
437 po_contents,
438 )
439
440 def test_template_comments(self):
441 """Template comment tags on the same line of other constructs (#19552)"""
442 # Test detection/end user reporting of old, incorrect templates
443 # translator comments syntax
444 with warnings.catch_warnings(record=True) as ws:
445 warnings.simplefilter("always")
446 management.call_command(
447 "makemessages", locale=[LOCALE], extensions=["thtml"], verbosity=0
448 )
449 self.assertEqual(len(ws), 3)
450 for w in ws:
451 self.assertTrue(issubclass(w.category, TranslatorCommentWarning))
452 self.assertRegex(
453 str(ws[0].message),
454 r"The translator-targeted comment 'Translators: ignored i18n "
455 r"comment #1' \(file templates[/\\]comments.thtml, line 4\) "
456 r"was ignored, because it wasn't the last item on the line\.",
457 )
458 self.assertRegex(
459 str(ws[1].message),
460 r"The translator-targeted comment 'Translators: ignored i18n "
461 r"comment #3' \(file templates[/\\]comments.thtml, line 6\) "
462 r"was ignored, because it wasn't the last item on the line\.",
463 )
464 self.assertRegex(
465 str(ws[2].message),
466 r"The translator-targeted comment 'Translators: ignored i18n "
467 r"comment #4' \(file templates[/\\]comments.thtml, line 8\) "
468 r"was ignored, because it wasn't the last item on the line\.",
469 )
470 # Now test .po file contents
471 self.assertTrue(os.path.exists(self.PO_FILE))
472 with open(self.PO_FILE) as fp:
473 po_contents = fp.read()
474
475 self.assertMsgId("Translatable literal #9a", po_contents)
476 self.assertNotIn("ignored comment #1", po_contents)
477
478 self.assertNotIn("Translators: ignored i18n comment #1", po_contents)
479 self.assertMsgId("Translatable literal #9b", po_contents)
480
481 self.assertNotIn("ignored i18n comment #2", po_contents)
482 self.assertNotIn("ignored comment #2", po_contents)
483 self.assertMsgId("Translatable literal #9c", po_contents)
484
485 self.assertNotIn("ignored comment #3", po_contents)
486 self.assertNotIn("ignored i18n comment #3", po_contents)
487 self.assertMsgId("Translatable literal #9d", po_contents)
488
489 self.assertNotIn("ignored comment #4", po_contents)
490 self.assertMsgId("Translatable literal #9e", po_contents)
491 self.assertNotIn("ignored comment #5", po_contents)
492
493 self.assertNotIn("ignored i18n comment #4", po_contents)
494 self.assertMsgId("Translatable literal #9f", po_contents)
495 self.assertIn("#. Translators: valid i18n comment #5", po_contents)
496
497 self.assertMsgId("Translatable literal #9g", po_contents)
498 self.assertIn("#. Translators: valid i18n comment #6", po_contents)
499 self.assertMsgId("Translatable literal #9h", po_contents)
500 self.assertIn("#. Translators: valid i18n comment #7", po_contents)
501 self.assertMsgId("Translatable literal #9i", po_contents)
502
503 self.assertRegex(po_contents, r"#\..+Translators: valid i18n comment #8")
504 self.assertRegex(po_contents, r"#\..+Translators: valid i18n comment #9")
505 self.assertMsgId("Translatable literal #9j", po_contents)
506
507 def test_makemessages_find_files(self):
508 """
509 find_files only discover files having the proper extensions.
510 """
511 cmd = MakeMessagesCommand()
512 cmd.ignore_patterns = ["CVS", ".*", "*~", "*.pyc"]
513 cmd.symlinks = False
514 cmd.domain = "django"
515 cmd.extensions = [".html", ".txt", ".py"]
516 cmd.verbosity = 0
517 cmd.locale_paths = []
518 cmd.default_locale_path = os.path.join(self.test_dir, "locale")
519 found_files = cmd.find_files(self.test_dir)
520 self.assertGreater(len(found_files), 1)
521 found_exts = {os.path.splitext(tfile.file)[1] for tfile in found_files}
522 self.assertEqual(found_exts.difference({".py", ".html", ".txt"}), set())
523
524 cmd.extensions = [".js"]
525 cmd.domain = "djangojs"
526 found_files = cmd.find_files(self.test_dir)
527 self.assertGreater(len(found_files), 1)
528 found_exts = {os.path.splitext(tfile.file)[1] for tfile in found_files}
529 self.assertEqual(found_exts.difference({".js"}), set())
530
531 @mock.patch("django.core.management.commands.makemessages.popen_wrapper")
532 def test_makemessages_gettext_version(self, mocked_popen_wrapper):
533 # "Normal" output:
534 mocked_popen_wrapper.return_value = (
535 "xgettext (GNU gettext-tools) 0.18.1\n"
536 "Copyright (C) 1995-1998, 2000-2010 Free Software Foundation, Inc.\n"
537 "License GPLv3+: GNU GPL version 3 or later "
538 "<http://gnu.org/licenses/gpl.html>\n"
539 "This is free software: you are free to change and redistribute it.\n"
540 "There is NO WARRANTY, to the extent permitted by law.\n"
541 "Written by Ulrich Drepper.\n",
542 "",
543 0,
544 )
545 cmd = MakeMessagesCommand()
546 self.assertEqual(cmd.gettext_version, (0, 18, 1))
547
548 # Version number with only 2 parts (#23788)
549 mocked_popen_wrapper.return_value = (
550 "xgettext (GNU gettext-tools) 0.17\n",
551 "",
552 0,
553 )
554 cmd = MakeMessagesCommand()
555 self.assertEqual(cmd.gettext_version, (0, 17))
556
557 # Bad version output
558 mocked_popen_wrapper.return_value = ("any other return value\n", "", 0)
559 cmd = MakeMessagesCommand()
560 with self.assertRaisesMessage(
561 CommandError, "Unable to get gettext version. Is it installed?"
562 ):
563 cmd.gettext_version
564
565 def test_po_file_encoding_when_updating(self):
566 """
567 Update of PO file doesn't corrupt it with non-UTF-8 encoding on Windows
568 (#23271).
569 """
570 BR_PO_BASE = "locale/pt_BR/LC_MESSAGES/django"
571 shutil.copyfile(BR_PO_BASE + ".pristine", BR_PO_BASE + ".po")
572 management.call_command("makemessages", locale=["pt_BR"], verbosity=0)
573 self.assertTrue(os.path.exists(BR_PO_BASE + ".po"))
574 with open(BR_PO_BASE + ".po", encoding="utf-8") as fp:
575 po_contents = fp.read()
576 self.assertMsgStr("Größe", po_contents)
577
578 def test_pot_charset_header_is_utf8(self):
579 """Content-Type: ... charset=CHARSET is replaced with charset=UTF-8"""
580 msgs = (
581 "# SOME DESCRIPTIVE TITLE.\n"
582 "# (some lines truncated as they are not relevant)\n"
583 '"Content-Type: text/plain; charset=CHARSET\\n"\n'
584 '"Content-Transfer-Encoding: 8bit\\n"\n'
585 "\n"
586 "#: somefile.py:8\n"
587 'msgid "mañana; charset=CHARSET"\n'
588 'msgstr ""\n'
589 )
590 with tempfile.NamedTemporaryFile() as pot_file:
591 pot_filename = pot_file.name
592 write_pot_file(pot_filename, msgs)
593 with open(pot_filename, encoding="utf-8") as fp:
594 pot_contents = fp.read()
595 self.assertIn("Content-Type: text/plain; charset=UTF-8", pot_contents)
596 self.assertIn("mañana; charset=CHARSET", pot_contents)
597
598
599 class JavaScriptExtractorTests(ExtractorTests):
600 PO_FILE = "locale/%s/LC_MESSAGES/djangojs.po" % LOCALE
601
602 def test_javascript_literals(self):
603 _, po_contents = self._run_makemessages(domain="djangojs")
604 self.assertMsgId("This literal should be included.", po_contents)
605 self.assertMsgId("gettext_noop should, too.", po_contents)
606 self.assertMsgId("This one as well.", po_contents)
607 self.assertMsgId(r"He said, \"hello\".", po_contents)
608 self.assertMsgId("okkkk", po_contents)
609 self.assertMsgId("TEXT", po_contents)
610 self.assertMsgId("It's at http://example.com", po_contents)
611 self.assertMsgId("String", po_contents)
612 self.assertMsgId(
613 "/* but this one will be too */ 'cause there is no way of telling...",
614 po_contents,
615 )
616 self.assertMsgId("foo", po_contents)
617 self.assertMsgId("bar", po_contents)
618 self.assertMsgId("baz", po_contents)
619 self.assertMsgId("quz", po_contents)
620 self.assertMsgId("foobar", po_contents)
621
622 def test_media_static_dirs_ignored(self):
623 """
624 Regression test for #23583.
625 """
626 with override_settings(
627 STATIC_ROOT=os.path.join(self.test_dir, "static/"),
628 MEDIA_ROOT=os.path.join(self.test_dir, "media_root/"),
629 ):
630 _, po_contents = self._run_makemessages(domain="djangojs")
631 self.assertMsgId(
632 "Static content inside app should be included.", po_contents
633 )
634 self.assertNotMsgId(
635 "Content from STATIC_ROOT should not be included", po_contents
636 )
637
638 @override_settings(STATIC_ROOT=None, MEDIA_ROOT="")
639 def test_default_root_settings(self):
640 """
641 Regression test for #23717.
642 """
643 _, po_contents = self._run_makemessages(domain="djangojs")
644 self.assertMsgId("Static content inside app should be included.", po_contents)
645
646 def test_i18n_catalog_ignored_when_invoked_for_django(self):
647 # Create target file so it exists in the filesystem and can be ignored.
648 # "invoked_for_django" is True when "conf/locale" folder exists.
649 os.makedirs(os.path.join("conf", "locale"))
650 i18n_catalog_js_dir = os.path.join(os.path.curdir, "views", "templates")
651 os.makedirs(i18n_catalog_js_dir)
652 open(os.path.join(i18n_catalog_js_dir, "i18n_catalog.js"), "w").close()
653
654 out, _ = self._run_makemessages(domain="djangojs")
655 self.assertIn(f"ignoring file i18n_catalog.js in {i18n_catalog_js_dir}", out)
656
657 def test_i18n_catalog_not_ignored_when_not_invoked_for_django(self):
658 # Create target file so it exists in the filesystem but is NOT ignored.
659 # "invoked_for_django" is False when "conf/locale" folder does not exist.
660 self.assertIs(os.path.exists(os.path.join("conf", "locale")), False)
661 i18n_catalog_js = os.path.join("views", "templates", "i18n_catalog.js")
662 os.makedirs(os.path.dirname(i18n_catalog_js))
663 open(i18n_catalog_js, "w").close()
664
665 out, _ = self._run_makemessages(domain="djangojs")
666 self.assertNotIn("ignoring file i18n_catalog.js", out)
667
668
669 class IgnoredExtractorTests(ExtractorTests):
670 def test_ignore_directory(self):
671 out, po_contents = self._run_makemessages(
672 ignore_patterns=[
673 os.path.join("ignore_dir", "*"),
674 ]
675 )
676 self.assertIn("ignoring directory ignore_dir", out)
677 self.assertMsgId("This literal should be included.", po_contents)
678 self.assertNotMsgId("This should be ignored.", po_contents)
679
680 def test_ignore_subdirectory(self):
681 out, po_contents = self._run_makemessages(
682 ignore_patterns=[
683 "templates/*/ignore.html",
684 "templates/subdir/*",
685 ]
686 )
687 self.assertIn("ignoring directory subdir", out)
688 self.assertNotMsgId("This subdir should be ignored too.", po_contents)
689
690 def test_ignore_file_patterns(self):
691 out, po_contents = self._run_makemessages(
692 ignore_patterns=[
693 "xxx_*",
694 ]
695 )
696 self.assertIn("ignoring file xxx_ignored.html", out)
697 self.assertNotMsgId("This should be ignored too.", po_contents)
698
699 def test_media_static_dirs_ignored(self):
700 with override_settings(
701 STATIC_ROOT=os.path.join(self.test_dir, "static/"),
702 MEDIA_ROOT=os.path.join(self.test_dir, "media_root/"),
703 ):
704 out, _ = self._run_makemessages()
705 self.assertIn("ignoring directory static", out)
706 self.assertIn("ignoring directory media_root", out)
707
708
709 class SymlinkExtractorTests(ExtractorTests):
710 def setUp(self):
711 super().setUp()
712 self.symlinked_dir = os.path.join(self.test_dir, "templates_symlinked")
713
714 def test_symlink(self):
715 if symlinks_supported():
716 os.symlink(os.path.join(self.test_dir, "templates"), self.symlinked_dir)
717 else:
718 self.skipTest(
719 "os.symlink() not available on this OS + Python version combination."
720 )
721 management.call_command(
722 "makemessages", locale=[LOCALE], verbosity=0, symlinks=True
723 )
724 self.assertTrue(os.path.exists(self.PO_FILE))
725 with open(self.PO_FILE) as fp:
726 po_contents = fp.read()
727 self.assertMsgId("This literal should be included.", po_contents)
728 self.assertLocationCommentPresent(
729 self.PO_FILE, None, "templates_symlinked", "test.html"
730 )
731
732
733 class CopyPluralFormsExtractorTests(ExtractorTests):
734 PO_FILE_ES = "locale/es/LC_MESSAGES/django.po"
735
736 def test_copy_plural_forms(self):
737 management.call_command("makemessages", locale=[LOCALE], verbosity=0)
738 self.assertTrue(os.path.exists(self.PO_FILE))
739 with open(self.PO_FILE) as fp:
740 po_contents = fp.read()
741 self.assertIn("Plural-Forms: nplurals=2; plural=(n != 1)", po_contents)
742
743 def test_override_plural_forms(self):
744 """Ticket #20311."""
745 management.call_command(
746 "makemessages", locale=["es"], extensions=["djtpl"], verbosity=0
747 )
748 self.assertTrue(os.path.exists(self.PO_FILE_ES))
749 with open(self.PO_FILE_ES, encoding="utf-8") as fp:
750 po_contents = fp.read()
751 found = re.findall(
752 r'^(?P<value>"Plural-Forms.+?\\n")\s*$',
753 po_contents,
754 re.MULTILINE | re.DOTALL,
755 )
756 self.assertEqual(1, len(found))
757
758 def test_translate_and_plural_blocktranslate_collision(self):
759 """
760 Ensures a correct workaround for the gettext bug when handling a literal
761 found inside a {% translate %} tag and also in another file inside a
762 {% blocktranslate %} with a plural (#17375).
763 """
764 management.call_command(
765 "makemessages", locale=[LOCALE], extensions=["html", "djtpl"], verbosity=0
766 )
767 self.assertTrue(os.path.exists(self.PO_FILE))
768 with open(self.PO_FILE) as fp:
769 po_contents = fp.read()
770 self.assertNotIn(
771 "#-#-#-#-# django.pot (PACKAGE VERSION) #-#-#-#-#\\n", po_contents
772 )
773 self.assertMsgId(
774 "First `translate`, then `blocktranslate` with a plural", po_contents
775 )
776 self.assertMsgIdPlural(
777 "Plural for a `translate` and `blocktranslate` collision case",
778 po_contents,
779 )
780
781
782 class NoWrapExtractorTests(ExtractorTests):
783 def test_no_wrap_enabled(self):
784 management.call_command(
785 "makemessages", locale=[LOCALE], verbosity=0, no_wrap=True
786 )
787 self.assertTrue(os.path.exists(self.PO_FILE))
788 with open(self.PO_FILE) as fp:
789 po_contents = fp.read()
790 self.assertMsgId(
791 "This literal should also be included wrapped or not wrapped "
792 "depending on the use of the --no-wrap option.",
793 po_contents,
794 )
795
796 def test_no_wrap_disabled(self):
797 management.call_command(
798 "makemessages", locale=[LOCALE], verbosity=0, no_wrap=False
799 )
800 self.assertTrue(os.path.exists(self.PO_FILE))
801 with open(self.PO_FILE) as fp:
802 po_contents = fp.read()
803 self.assertMsgId(
804 '""\n"This literal should also be included wrapped or not '
805 'wrapped depending on the "\n"use of the --no-wrap option."',
806 po_contents,
807 use_quotes=False,
808 )
809
810
811 class LocationCommentsTests(ExtractorTests):
812 def test_no_location_enabled(self):
813 """Behavior is correct if --no-location switch is specified. See #16903."""
814 management.call_command(
815 "makemessages", locale=[LOCALE], verbosity=0, no_location=True
816 )
817 self.assertTrue(os.path.exists(self.PO_FILE))
818 self.assertLocationCommentNotPresent(self.PO_FILE, None, "test.html")
819
820 def test_no_location_disabled(self):
821 """Behavior is correct if --no-location switch isn't specified."""
822 management.call_command(
823 "makemessages", locale=[LOCALE], verbosity=0, no_location=False
824 )
825 self.assertTrue(os.path.exists(self.PO_FILE))
826 # #16903 -- Standard comment with source file relative path should be present
827 self.assertLocationCommentPresent(
828 self.PO_FILE, "Translatable literal #6b", "templates", "test.html"
829 )
830
831 def test_location_comments_for_templatized_files(self):
832 """
833 Ensure no leaky paths in comments, e.g. #: path\to\file.html.py:123
834 Refs #21209/#26341.
835 """
836 management.call_command("makemessages", locale=[LOCALE], verbosity=0)
837 self.assertTrue(os.path.exists(self.PO_FILE))
838 with open(self.PO_FILE) as fp:
839 po_contents = fp.read()
840 self.assertMsgId("#: templates/test.html.py", po_contents)
841 self.assertLocationCommentNotPresent(self.PO_FILE, None, ".html.py")
842 self.assertLocationCommentPresent(self.PO_FILE, 5, "templates", "test.html")
843
844 def test_add_location_full(self):
845 """makemessages --add-location=full"""
846 management.call_command(
847 "makemessages", locale=[LOCALE], verbosity=0, add_location="full"
848 )
849 self.assertTrue(os.path.exists(self.PO_FILE))
850 # Comment with source file relative path and line number is present.
851 self.assertLocationCommentPresent(
852 self.PO_FILE, "Translatable literal #6b", "templates", "test.html"
853 )
854
855 def test_add_location_file(self):
856 """makemessages --add-location=file"""
857 management.call_command(
858 "makemessages", locale=[LOCALE], verbosity=0, add_location="file"
859 )
860 self.assertTrue(os.path.exists(self.PO_FILE))
861 # Comment with source file relative path is present.
862 self.assertLocationCommentPresent(self.PO_FILE, None, "templates", "test.html")
863 # But it should not contain the line number.
864 self.assertLocationCommentNotPresent(
865 self.PO_FILE, "Translatable literal #6b", "templates", "test.html"
866 )
867
868 def test_add_location_never(self):
869 """makemessages --add-location=never"""
870 management.call_command(
871 "makemessages", locale=[LOCALE], verbosity=0, add_location="never"
872 )
873 self.assertTrue(os.path.exists(self.PO_FILE))
874 self.assertLocationCommentNotPresent(self.PO_FILE, None, "test.html")
875
876
877 class NoObsoleteExtractorTests(ExtractorTests):
878 work_subdir = "obsolete_translations"
879
880 def test_no_obsolete(self):
881 management.call_command(
882 "makemessages", locale=[LOCALE], verbosity=0, no_obsolete=True
883 )
884 self.assertIs(os.path.exists(self.PO_FILE), True)
885 with open(self.PO_FILE) as fp:
886 po_contents = fp.read()
887 self.assertNotIn('#~ msgid "Obsolete string."', po_contents)
888 self.assertNotIn('#~ msgstr "Translated obsolete string."', po_contents)
889 self.assertMsgId("This is a translatable string.", po_contents)
890 self.assertMsgStr("This is a translated string.", po_contents)
891
892
893 class KeepPotFileExtractorTests(ExtractorTests):
894 POT_FILE = "locale/django.pot"
895
896 def test_keep_pot_disabled_by_default(self):
897 management.call_command("makemessages", locale=[LOCALE], verbosity=0)
898 self.assertFalse(os.path.exists(self.POT_FILE))
899
900 def test_keep_pot_explicitly_disabled(self):
901 management.call_command(
902 "makemessages", locale=[LOCALE], verbosity=0, keep_pot=False
903 )
904 self.assertFalse(os.path.exists(self.POT_FILE))
905
906 def test_keep_pot_enabled(self):
907 management.call_command(
908 "makemessages", locale=[LOCALE], verbosity=0, keep_pot=True
909 )
910 self.assertTrue(os.path.exists(self.POT_FILE))
911
912
913 class MultipleLocaleExtractionTests(ExtractorTests):
914 PO_FILE_PT = "locale/pt/LC_MESSAGES/django.po"
915 PO_FILE_DE = "locale/de/LC_MESSAGES/django.po"
916 PO_FILE_KO = "locale/ko/LC_MESSAGES/django.po"
917 LOCALES = ["pt", "de", "ch"]
918
919 def test_multiple_locales(self):
920 management.call_command("makemessages", locale=["pt", "de"], verbosity=0)
921 self.assertTrue(os.path.exists(self.PO_FILE_PT))
922 self.assertTrue(os.path.exists(self.PO_FILE_DE))
923
924 def test_all_locales(self):
925 """
926 When the `locale` flag is absent, all dirs from the parent locale dir
927 are considered as language directories, except if the directory doesn't
928 start with two letters (which excludes __pycache__, .gitignore, etc.).
929 """
930 os.mkdir(os.path.join("locale", "_do_not_pick"))
931 # Excluding locales that do not compile
932 management.call_command("makemessages", exclude=["ja", "es_AR"], verbosity=0)
933 self.assertTrue(os.path.exists(self.PO_FILE_KO))
934 self.assertFalse(os.path.exists("locale/_do_not_pick/LC_MESSAGES/django.po"))
935
936
937 class ExcludedLocaleExtractionTests(ExtractorTests):
938 work_subdir = "exclude"
939
940 LOCALES = ["en", "fr", "it"]
941 PO_FILE = "locale/%s/LC_MESSAGES/django.po"
942
943 def _set_times_for_all_po_files(self):
944 """
945 Set access and modification times to the Unix epoch time for all the .po files.
946 """
947 for locale in self.LOCALES:
948 os.utime(self.PO_FILE % locale, (0, 0))
949
950 def setUp(self):
951 super().setUp()
952 copytree("canned_locale", "locale")
953 self._set_times_for_all_po_files()
954
955 def test_command_help(self):
956 with captured_stdout(), captured_stderr():
957 # `call_command` bypasses the parser; by calling
958 # `execute_from_command_line` with the help subcommand we
959 # ensure that there are no issues with the parser itself.
960 execute_from_command_line(["django-admin", "help", "makemessages"])
961
962 def test_one_locale_excluded(self):
963 management.call_command("makemessages", exclude=["it"], verbosity=0)
964 self.assertRecentlyModified(self.PO_FILE % "en")
965 self.assertRecentlyModified(self.PO_FILE % "fr")
966 self.assertNotRecentlyModified(self.PO_FILE % "it")
967
968 def test_multiple_locales_excluded(self):
969 management.call_command("makemessages", exclude=["it", "fr"], verbosity=0)
970 self.assertRecentlyModified(self.PO_FILE % "en")
971 self.assertNotRecentlyModified(self.PO_FILE % "fr")
972 self.assertNotRecentlyModified(self.PO_FILE % "it")
973
974 def test_one_locale_excluded_with_locale(self):
975 management.call_command(
976 "makemessages", locale=["en", "fr"], exclude=["fr"], verbosity=0
977 )
978 self.assertRecentlyModified(self.PO_FILE % "en")
979 self.assertNotRecentlyModified(self.PO_FILE % "fr")
980 self.assertNotRecentlyModified(self.PO_FILE % "it")
981
982 def test_multiple_locales_excluded_with_locale(self):
983 management.call_command(
984 "makemessages", locale=["en", "fr", "it"], exclude=["fr", "it"], verbosity=0
985 )
986 self.assertRecentlyModified(self.PO_FILE % "en")
987 self.assertNotRecentlyModified(self.PO_FILE % "fr")
988 self.assertNotRecentlyModified(self.PO_FILE % "it")
989
990
991 class CustomLayoutExtractionTests(ExtractorTests):
992 work_subdir = "project_dir"
993
994 def test_no_locale_raises(self):
995 msg = (
996 "Unable to find a locale path to store translations for file "
997 "__init__.py. Make sure the 'locale' directory exists in an app "
998 "or LOCALE_PATHS setting is set."
999 )
1000 with self.assertRaisesMessage(management.CommandError, msg):
1001 management.call_command("makemessages", locale=[LOCALE], verbosity=0)
1002 # Working files are cleaned up on an error.
1003 self.assertFalse(os.path.exists("./app_no_locale/test.html.py"))
1004
1005 def test_project_locale_paths(self):
1006 self._test_project_locale_paths(os.path.join(self.test_dir, "project_locale"))
1007
1008 def test_project_locale_paths_pathlib(self):
1009 self._test_project_locale_paths(Path(self.test_dir) / "project_locale")
1010
1011 def _test_project_locale_paths(self, locale_path):
1012 """
1013 * translations for an app containing a locale folder are stored in that folder
1014 * translations outside of that app are in LOCALE_PATHS[0]
1015 """
1016 with override_settings(LOCALE_PATHS=[locale_path]):
1017 management.call_command("makemessages", locale=[LOCALE], verbosity=0)
1018 project_de_locale = os.path.join(
1019 self.test_dir, "project_locale", "de", "LC_MESSAGES", "django.po"
1020 )
1021 app_de_locale = os.path.join(
1022 self.test_dir,
1023 "app_with_locale",
1024 "locale",
1025 "de",
1026 "LC_MESSAGES",
1027 "django.po",
1028 )
1029 self.assertTrue(os.path.exists(project_de_locale))
1030 self.assertTrue(os.path.exists(app_de_locale))
1031
1032 with open(project_de_locale) as fp:
1033 po_contents = fp.read()
1034 self.assertMsgId("This app has no locale directory", po_contents)
1035 self.assertMsgId("This is a project-level string", po_contents)
1036 with open(app_de_locale) as fp:
1037 po_contents = fp.read()
1038 self.assertMsgId("This app has a locale directory", po_contents)
1039
1040
1041 @skipUnless(has_xgettext, "xgettext is mandatory for extraction tests")
1042 class NoSettingsExtractionTests(AdminScriptTestCase):
1043 def test_makemessages_no_settings(self):
1044 out, err = self.run_django_admin(["makemessages", "-l", "en", "-v", "0"])
1045 self.assertNoOutput(err)
1046 self.assertNoOutput(out)
1047
1048
1049 class UnchangedPoExtractionTests(ExtractorTests):
1050 work_subdir = "unchanged"
1051
1052 def setUp(self):
1053 super().setUp()
1054 po_file = Path(self.PO_FILE)
1055 po_file_tmp = Path(self.PO_FILE + ".tmp")
1056 if os.name == "nt":
1057 # msgmerge outputs Windows style paths on Windows.
1058 po_contents = po_file_tmp.read_text().replace(
1059 "#: __init__.py",
1060 "#: .\\__init__.py",
1061 )
1062 po_file.write_text(po_contents)
1063 else:
1064 po_file_tmp.rename(po_file)
1065 self.original_po_contents = po_file.read_text()
1066
1067 def test_po_remains_unchanged(self):
1068 """PO files are unchanged unless there are new changes."""
1069 _, po_contents = self._run_makemessages()
1070 self.assertEqual(po_contents, self.original_po_contents)
1071
1072 def test_po_changed_with_new_strings(self):
1073 """PO files are updated when new changes are detected."""
1074 Path("models.py.tmp").rename("models.py")
1075 _, po_contents = self._run_makemessages()
1076 self.assertNotEqual(po_contents, self.original_po_contents)
1077 self.assertMsgId(
1078 "This is a hitherto undiscovered translatable string.",
1079 po_contents,
1080 )