Debian Codegraph @

Lib/test/test_c_locale_coercion.py

raw
1 # Tests the attempted automatic coercion of the C locale to a UTF-8 locale
2
3 import locale
4 import os
5 import subprocess
6 import sys
7 import sysconfig
8 import unittest
9 from collections import namedtuple
10
11 from test import support
12 from test.support.script_helper import run_python_until_end
13
14
15 # Set the list of ways we expect to be able to ask for the "C" locale.
16 # 'invalid.ascii' is an invalid LOCALE name and so should get turned in to the
17 # default locale, which is traditionally C.
18 EXPECTED_C_LOCALE_EQUIVALENTS = ["C", "invalid.ascii"]
19
20 # Set our expectation for the default encoding used in the C locale
21 # for the filesystem encoding and the standard streams
22 EXPECTED_C_LOCALE_STREAM_ENCODING = "ascii"
23 EXPECTED_C_LOCALE_FS_ENCODING = "ascii"
24
25 # Set our expectation for the default locale used when none is specified
26 DEFAULT_LOCALE_IS_C = True
27 EXPECT_COERCION_IN_DEFAULT_LOCALE = True
28
29 TARGET_LOCALES = ["C.UTF-8", "C.utf8", "UTF-8"]
30
31 # Apply some platform dependent overrides
32 if sys.platform == "android":
33 # Android defaults to using UTF-8 for all system interfaces
34 EXPECTED_C_LOCALE_STREAM_ENCODING = "utf-8"
35 EXPECTED_C_LOCALE_FS_ENCODING = "utf-8"
36 elif support.linked_to_musl():
37 # MUSL defaults to utf-8 unless the C locale is set explicitly.
38 EXPECTED_C_LOCALE_EQUIVALENTS = ["C"]
39 DEFAULT_LOCALE_IS_C = False
40 DEFAULT_ENCODING = 'utf-8'
41 EXPECT_COERCION_IN_DEFAULT_LOCALE = False
42 elif sys.platform.startswith("aix"):
43 # AIX uses iso8859-1 in the C locale, other *nix platforms use ASCII
44 EXPECTED_C_LOCALE_STREAM_ENCODING = "iso8859-1"
45 EXPECTED_C_LOCALE_FS_ENCODING = "iso8859-1"
46 elif sys.platform == "darwin":
47 # FS encoding is UTF-8 on macOS
48 EXPECTED_C_LOCALE_FS_ENCODING = "utf-8"
49 elif sys.platform == "cygwin":
50 # Cygwin defaults to using C.UTF-8
51 # TODO: Work out a robust dynamic test for this that doesn't rely on
52 # CPython's own locale handling machinery
53 EXPECT_COERCION_IN_DEFAULT_LOCALE = False
54 elif sys.platform == "vxworks":
55 # VxWorks defaults to using UTF-8 for all system interfaces
56 EXPECTED_C_LOCALE_STREAM_ENCODING = "utf-8"
57 EXPECTED_C_LOCALE_FS_ENCODING = "utf-8"
58 if sys.platform.startswith("linux"):
59 # Linux recognizes POSIX as a synonym for C. Python will always coerce
60 # if the locale is set to POSIX, but not all platforms will use the
61 # C locale encodings if POSIX is set, so we'll only test it on linux.
62 EXPECTED_C_LOCALE_EQUIVALENTS.append("POSIX")
63
64 # Note that the above expectations are still wrong in some cases, such as:
65 # * Windows when PYTHONLEGACYWINDOWSFSENCODING is set
66 # * Any platform other than AIX that uses latin-1 in the C locale
67 # * Any Linux distro where POSIX isn't a simple alias for the C locale
68 # * Any Linux distro where the default locale is something other than "C"
69 #
70 # Options for dealing with this:
71 # * Don't set the PY_COERCE_C_LOCALE preprocessor definition on
72 # such platforms (e.g. it isn't set on Windows)
73 # * Fix the test expectations to match the actual platform behaviour
74
75 # In order to get the warning messages to match up as expected, the candidate
76 # order here must much the target locale order in Python/pylifecycle.c
77 _C_UTF8_LOCALES = ("C.UTF-8", "C.utf8", "UTF-8")
78
79 # There's no reliable cross-platform way of checking locale alias
80 # lists, so the only way of knowing which of these locales will work
81 # is to try them with locale.setlocale(). We do that in a subprocess
82 # in setUpModule() below to avoid altering the locale of the test runner.
83 #
84 # If the relevant locale module attributes exist, and we're not on a platform
85 # where we expect it to always succeed, we also check that
86 # `locale.nl_langinfo(locale.CODESET)` works, as if it fails, the interpreter
87 # will skip locale coercion for that particular target locale
88 _check_nl_langinfo_CODESET = bool(
89 sys.platform not in ("darwin", "linux") and
90 hasattr(locale, "nl_langinfo") and
91 hasattr(locale, "CODESET")
92 )
93
94 def _set_locale_in_subprocess(locale_name):
95 cmd_fmt = "import locale; print(locale.setlocale(locale.LC_CTYPE, '{}'))"
96 if _check_nl_langinfo_CODESET:
97 # If there's no valid CODESET, we expect coercion to be skipped
98 cmd_fmt += "; import sys; sys.exit(not locale.nl_langinfo(locale.CODESET))"
99 cmd = cmd_fmt.format(locale_name)
100 result, py_cmd = run_python_until_end("-c", cmd, PYTHONCOERCECLOCALE='')
101 return result.rc == 0
102
103
104
105 _fields = "fsencoding stdin_info stdout_info stderr_info lang lc_ctype lc_all"
106 _EncodingDetails = namedtuple("EncodingDetails", _fields)
107
108 class EncodingDetails(_EncodingDetails):
109 # XXX (ncoghlan): Using JSON for child state reporting may be less fragile
110 CHILD_PROCESS_SCRIPT = ";".join([
111 "import sys, os",
112 "print(sys.getfilesystemencoding())",
113 "print(sys.stdin.encoding + ':' + sys.stdin.errors)",
114 "print(sys.stdout.encoding + ':' + sys.stdout.errors)",
115 "print(sys.stderr.encoding + ':' + sys.stderr.errors)",
116 "print(os.environ.get('LANG', 'not set'))",
117 "print(os.environ.get('LC_CTYPE', 'not set'))",
118 "print(os.environ.get('LC_ALL', 'not set'))",
119 ])
120
121 @classmethod
122 def get_expected_details(cls, coercion_expected, fs_encoding, stream_encoding, stream_errors, env_vars):
123 """Returns expected child process details for a given encoding"""
124 _stream = stream_encoding + ":{}"
125 if stream_errors is None:
126 # stdin and stdout should use surrogateescape either because the
127 # coercion triggered, or because the C locale was detected
128 stream_errors = "surrogateescape"
129
130 stream_info = [_stream.format(stream_errors)] * 2
131
132 # stderr should always use backslashreplace
133 stream_info.append(_stream.format("backslashreplace"))
134 expected_lang = env_vars.get("LANG", "not set")
135 if coercion_expected:
136 expected_lc_ctype = CLI_COERCION_TARGET
137 else:
138 expected_lc_ctype = env_vars.get("LC_CTYPE", "not set")
139 expected_lc_all = env_vars.get("LC_ALL", "not set")
140 env_info = expected_lang, expected_lc_ctype, expected_lc_all
141 return dict(cls(fs_encoding, *stream_info, *env_info)._asdict())
142
143 @classmethod
144 def get_child_details(cls, env_vars):
145 """Retrieves fsencoding and standard stream details from a child process
146
147 Returns (encoding_details, stderr_lines):
148
149 - encoding_details: EncodingDetails for eager decoding
150 - stderr_lines: result of calling splitlines() on the stderr output
151
152 The child is run in isolated mode if the current interpreter supports
153 that.
154 """
155 result, py_cmd = run_python_until_end(
156 "-X", "utf8=0", "-c", cls.CHILD_PROCESS_SCRIPT,
157 **env_vars
158 )
159 if not result.rc == 0:
160 result.fail(py_cmd)
161 # All subprocess outputs in this test case should be pure ASCII
162 stdout_lines = result.out.decode("ascii").splitlines()
163 child_encoding_details = dict(cls(*stdout_lines)._asdict())
164 stderr_lines = result.err.decode("ascii").rstrip().splitlines()
165 return child_encoding_details, stderr_lines
166
167
168 # Details of the shared library warning emitted at runtime
169 LEGACY_LOCALE_WARNING = (
170 "Python runtime initialized with LC_CTYPE=C (a locale with default ASCII "
171 "encoding), which may cause Unicode compatibility problems. Using C.UTF-8, "
172 "C.utf8, or UTF-8 (if available) as alternative Unicode-compatible "
173 "locales is recommended."
174 )
175
176 # Details of the CLI locale coercion warning emitted at runtime
177 CLI_COERCION_WARNING_FMT = (
178 "Python detected LC_CTYPE=C: LC_CTYPE coerced to {} (set another locale "
179 "or PYTHONCOERCECLOCALE=0 to disable this locale coercion behavior)."
180 )
181
182
183 AVAILABLE_TARGETS = None
184 CLI_COERCION_TARGET = None
185 CLI_COERCION_WARNING = None
186
187 def setUpModule():
188 global AVAILABLE_TARGETS
189 global CLI_COERCION_TARGET
190 global CLI_COERCION_WARNING
191
192 if AVAILABLE_TARGETS is not None:
193 # initialization already done
194 return
195 AVAILABLE_TARGETS = []
196
197 # Find the target locales available in the current system
198 for target_locale in _C_UTF8_LOCALES:
199 if _set_locale_in_subprocess(target_locale):
200 AVAILABLE_TARGETS.append(target_locale)
201
202 if AVAILABLE_TARGETS:
203 # Coercion is expected to use the first available target locale
204 CLI_COERCION_TARGET = AVAILABLE_TARGETS[0]
205 CLI_COERCION_WARNING = CLI_COERCION_WARNING_FMT.format(CLI_COERCION_TARGET)
206
207 if support.verbose:
208 print(f"AVAILABLE_TARGETS = {AVAILABLE_TARGETS!r}")
209 print(f"EXPECTED_C_LOCALE_EQUIVALENTS = {EXPECTED_C_LOCALE_EQUIVALENTS!r}")
210 print(f"EXPECTED_C_LOCALE_STREAM_ENCODING = {EXPECTED_C_LOCALE_STREAM_ENCODING!r}")
211 print(f"EXPECTED_C_LOCALE_FS_ENCODING = {EXPECTED_C_LOCALE_FS_ENCODING!r}")
212 print(f"EXPECT_COERCION_IN_DEFAULT_LOCALE = {EXPECT_COERCION_IN_DEFAULT_LOCALE!r}")
213 print(f"_C_UTF8_LOCALES = {_C_UTF8_LOCALES!r}")
214 print(f"_check_nl_langinfo_CODESET = {_check_nl_langinfo_CODESET!r}")
215
216
217 class _LocaleHandlingTestCase(unittest.TestCase):
218 # Base class to check expected locale handling behaviour
219
220 def _check_child_encoding_details(self,
221 env_vars,
222 expected_fs_encoding,
223 expected_stream_encoding,
224 expected_stream_errors,
225 expected_warnings,
226 coercion_expected):
227 """Check the C locale handling for the given process environment
228
229 Parameters:
230 expected_fs_encoding: expected sys.getfilesystemencoding() result
231 expected_stream_encoding: expected encoding for standard streams
232 expected_warning: stderr output to expect (if any)
233 """
234 result = EncodingDetails.get_child_details(env_vars)
235 encoding_details, stderr_lines = result
236 expected_details = EncodingDetails.get_expected_details(
237 coercion_expected,
238 expected_fs_encoding,
239 expected_stream_encoding,
240 expected_stream_errors,
241 env_vars
242 )
243 self.assertEqual(encoding_details, expected_details)
244 if expected_warnings is None:
245 expected_warnings = []
246 self.assertEqual(stderr_lines, expected_warnings)
247
248
249 class LocaleConfigurationTests(_LocaleHandlingTestCase):
250 # Test explicit external configuration via the process environment
251
252 @classmethod
253 def setUpClass(cls):
254 # This relies on setUpModule() having been run, so it can't be
255 # handled via the @unittest.skipUnless decorator
256 if not AVAILABLE_TARGETS:
257 raise unittest.SkipTest("No C-with-UTF-8 locale available")
258
259 def test_external_target_locale_configuration(self):
260
261 # Explicitly setting a target locale should give the same behaviour as
262 # is seen when implicitly coercing to that target locale
263 self.maxDiff = None
264
265 expected_fs_encoding = "utf-8"
266 expected_stream_encoding = "utf-8"
267
268 base_var_dict = {
269 "LANG": "",
270 "LC_CTYPE": "",
271 "LC_ALL": "",
272 "PYTHONCOERCECLOCALE": "",
273 "PYTHONIOENCODING": "",
274 }
275 for env_var in ("LANG", "LC_CTYPE"):
276 for locale_to_set in AVAILABLE_TARGETS:
277 # XXX (ncoghlan): LANG=UTF-8 doesn't appear to work as
278 # expected, so skip that combination for now
279 # See https://bugs.python.org/issue30672 for discussion
280 if env_var == "LANG" and locale_to_set == "UTF-8":
281 continue
282
283 with self.subTest(env_var=env_var,
284 configured_locale=locale_to_set):
285 var_dict = base_var_dict.copy()
286 var_dict[env_var] = locale_to_set
287 self._check_child_encoding_details(var_dict,
288 expected_fs_encoding,
289 expected_stream_encoding,
290 expected_stream_errors=None,
291 expected_warnings=None,
292 coercion_expected=False)
293
294 def test_with_ioencoding(self):
295 # Explicitly setting a target locale should give the same behaviour as
296 # is seen when implicitly coercing to that target locale
297 self.maxDiff = None
298
299 expected_fs_encoding = "utf-8"
300 expected_stream_encoding = "utf-8"
301
302 base_var_dict = {
303 "LANG": "",
304 "LC_CTYPE": "",
305 "LC_ALL": "",
306 "PYTHONCOERCECLOCALE": "",
307 "PYTHONIOENCODING": "UTF-8",
308 }
309 for env_var in ("LANG", "LC_CTYPE"):
310 for locale_to_set in AVAILABLE_TARGETS:
311 # XXX (ncoghlan): LANG=UTF-8 doesn't appear to work as
312 # expected, so skip that combination for now
313 # See https://bugs.python.org/issue30672 for discussion
314 if env_var == "LANG" and locale_to_set == "UTF-8":
315 continue
316
317 with self.subTest(env_var=env_var,
318 configured_locale=locale_to_set):
319 var_dict = base_var_dict.copy()
320 var_dict[env_var] = locale_to_set
321 self._check_child_encoding_details(var_dict,
322 expected_fs_encoding,
323 expected_stream_encoding,
324 expected_stream_errors="strict",
325 expected_warnings=None,
326 coercion_expected=False)
327
328 @support.cpython_only
329 @unittest.skipUnless(sysconfig.get_config_var("PY_COERCE_C_LOCALE"),
330 "C locale coercion disabled at build time")
331 class LocaleCoercionTests(_LocaleHandlingTestCase):
332 # Test implicit reconfiguration of the environment during CLI startup
333
334 def _check_c_locale_coercion(self,
335 fs_encoding, stream_encoding,
336 coerce_c_locale,
337 expected_warnings=None,
338 coercion_expected=True,
339 **extra_vars):
340 """Check the C locale handling for various configurations
341
342 Parameters:
343 fs_encoding: expected sys.getfilesystemencoding() result
344 stream_encoding: expected encoding for standard streams
345 coerce_c_locale: setting to use for PYTHONCOERCECLOCALE
346 None: don't set the variable at all
347 str: the value set in the child's environment
348 expected_warnings: expected warning lines on stderr
349 extra_vars: additional environment variables to set in subprocess
350 """
351 self.maxDiff = None
352
353 if not AVAILABLE_TARGETS:
354 # Locale coercion is disabled when there aren't any target locales
355 fs_encoding = EXPECTED_C_LOCALE_FS_ENCODING
356 stream_encoding = EXPECTED_C_LOCALE_STREAM_ENCODING
357 coercion_expected = False
358 if expected_warnings:
359 expected_warnings = [LEGACY_LOCALE_WARNING]
360
361 base_var_dict = {
362 "LANG": "",
363 "LC_CTYPE": "",
364 "LC_ALL": "",
365 "PYTHONCOERCECLOCALE": "",
366 "PYTHONIOENCODING": "",
367 }
368 base_var_dict.update(extra_vars)
369 if coerce_c_locale is not None:
370 base_var_dict["PYTHONCOERCECLOCALE"] = coerce_c_locale
371
372 # Check behaviour for the default locale
373 _fs_encoding = fs_encoding
374 _stream_encoding = stream_encoding
375 if not DEFAULT_LOCALE_IS_C and 'LC_ALL' not in extra_vars:
376 _fs_encoding = _stream_encoding = DEFAULT_ENCODING
377 with self.subTest(default_locale=True,
378 PYTHONCOERCECLOCALE=coerce_c_locale):
379 if (EXPECT_COERCION_IN_DEFAULT_LOCALE
380 or (not DEFAULT_LOCALE_IS_C and 'LC_ALL' in extra_vars)):
381 _expected_warnings = expected_warnings
382 _coercion_expected = coercion_expected
383 else:
384 _expected_warnings = None
385 _coercion_expected = False
386 # On Android CLI_COERCION_WARNING is not printed when all the
387 # locale environment variables are undefined or empty. When
388 # this code path is run with environ['LC_ALL'] == 'C', then
389 # LEGACY_LOCALE_WARNING is printed.
390 if (support.is_android and
391 _expected_warnings == [CLI_COERCION_WARNING]):
392 _expected_warnings = None
393 self._check_child_encoding_details(base_var_dict,
394 _fs_encoding,
395 _stream_encoding,
396 None,
397 _expected_warnings,
398 _coercion_expected)
399
400 # Check behaviour for explicitly configured locales
401 for locale_to_set in EXPECTED_C_LOCALE_EQUIVALENTS:
402 for env_var in ("LANG", "LC_CTYPE"):
403 with self.subTest(env_var=env_var,
404 nominal_locale=locale_to_set,
405 PYTHONCOERCECLOCALE=coerce_c_locale,
406 PYTHONIOENCODING=""):
407 var_dict = base_var_dict.copy()
408 var_dict[env_var] = locale_to_set
409 # Check behaviour on successful coercion
410 self._check_child_encoding_details(var_dict,
411 fs_encoding,
412 stream_encoding,
413 None,
414 expected_warnings,
415 coercion_expected)
416
417 def test_PYTHONCOERCECLOCALE_not_set(self):
418 # This should coerce to the first available target locale by default
419 self._check_c_locale_coercion("utf-8", "utf-8", coerce_c_locale=None)
420
421 def test_PYTHONCOERCECLOCALE_not_zero(self):
422 # *Any* string other than "0" is considered "set" for our purposes
423 # and hence should result in the locale coercion being enabled
424 for setting in ("", "1", "true", "false"):
425 self._check_c_locale_coercion("utf-8", "utf-8", coerce_c_locale=setting)
426
427 def test_PYTHONCOERCECLOCALE_set_to_warn(self):
428 # PYTHONCOERCECLOCALE=warn enables runtime warnings for legacy locales
429 self._check_c_locale_coercion("utf-8", "utf-8",
430 coerce_c_locale="warn",
431 expected_warnings=[CLI_COERCION_WARNING])
432
433
434 def test_PYTHONCOERCECLOCALE_set_to_zero(self):
435 # The setting "0" should result in the locale coercion being disabled
436 self._check_c_locale_coercion(EXPECTED_C_LOCALE_FS_ENCODING,
437 EXPECTED_C_LOCALE_STREAM_ENCODING,
438 coerce_c_locale="0",
439 coercion_expected=False)
440 # Setting LC_ALL=C shouldn't make any difference to the behaviour
441 self._check_c_locale_coercion(EXPECTED_C_LOCALE_FS_ENCODING,
442 EXPECTED_C_LOCALE_STREAM_ENCODING,
443 coerce_c_locale="0",
444 LC_ALL="C",
445 coercion_expected=False)
446
447 def test_LC_ALL_set_to_C(self):
448 # Setting LC_ALL should render the locale coercion ineffective
449 self._check_c_locale_coercion(EXPECTED_C_LOCALE_FS_ENCODING,
450 EXPECTED_C_LOCALE_STREAM_ENCODING,
451 coerce_c_locale=None,
452 LC_ALL="C",
453 coercion_expected=False)
454 # And result in a warning about a lack of locale compatibility
455 self._check_c_locale_coercion(EXPECTED_C_LOCALE_FS_ENCODING,
456 EXPECTED_C_LOCALE_STREAM_ENCODING,
457 coerce_c_locale="warn",
458 LC_ALL="C",
459 expected_warnings=[LEGACY_LOCALE_WARNING],
460 coercion_expected=False)
461
462 def test_PYTHONCOERCECLOCALE_set_to_one(self):
463 # skip the test if the LC_CTYPE locale is C or coerced
464 old_loc = locale.setlocale(locale.LC_CTYPE, None)
465 self.addCleanup(locale.setlocale, locale.LC_CTYPE, old_loc)
466 try:
467 loc = locale.setlocale(locale.LC_CTYPE, "")
468 except locale.Error as e:
469 self.skipTest(str(e))
470 if loc == "C":
471 self.skipTest("test requires LC_CTYPE locale different than C")
472 if loc in TARGET_LOCALES :
473 self.skipTest("coerced LC_CTYPE locale: %s" % loc)
474
475 # bpo-35336: PYTHONCOERCECLOCALE=1 must not coerce the LC_CTYPE locale
476 # if it's not equal to "C"
477 code = 'import locale; print(locale.setlocale(locale.LC_CTYPE, None))'
478 env = dict(os.environ, PYTHONCOERCECLOCALE='1')
479 cmd = subprocess.run([sys.executable, '-c', code],
480 stdout=subprocess.PIPE,
481 env=env,
482 text=True)
483 self.assertEqual(cmd.stdout.rstrip(), loc)
484
485
486 def tearDownModule():
487 support.reap_children()
488
489
490 if __name__ == "__main__":
491 unittest.main()