| 1 |
import contextlib |
| 2 |
import faulthandler |
| 3 |
import locale |
| 4 |
import math |
| 5 |
import os.path |
| 6 |
import platform |
| 7 |
import random |
| 8 |
import re |
| 9 |
import shlex |
| 10 |
import subprocess |
| 11 |
import sys |
| 12 |
import sysconfig |
| 13 |
import tempfile |
| 14 |
import textwrap |
| 15 |
from collections.abc import Callable |
| 16 |
|
| 17 |
from test import support |
| 18 |
from test.support import os_helper |
| 19 |
from test.support import threading_helper |
| 20 |
|
| 21 |
|
| 22 |
# All temporary files and temporary directories created by libregrtest should |
| 23 |
# use TMP_PREFIX so cleanup_temp_dir() can remove them all. |
| 24 |
TMP_PREFIX = 'test_python_' |
| 25 |
WORK_DIR_PREFIX = TMP_PREFIX |
| 26 |
WORKER_WORK_DIR_PREFIX = WORK_DIR_PREFIX + 'worker_' |
| 27 |
|
| 28 |
# bpo-38203: Maximum delay in seconds to exit Python (call Py_Finalize()). |
| 29 |
# Used to protect against threading._shutdown() hang. |
| 30 |
# Must be smaller than buildbot "1200 seconds without output" limit. |
| 31 |
EXIT_TIMEOUT = 120.0 |
| 32 |
|
| 33 |
|
| 34 |
ALL_RESOURCES = ('audio', 'console', 'curses', 'largefile', 'network', |
| 35 |
'decimal', 'cpu', 'subprocess', 'urlfetch', 'gui', 'walltime') |
| 36 |
|
| 37 |
# Other resources excluded from --use=all: |
| 38 |
# |
| 39 |
# - extralagefile (ex: test_zipfile64): really too slow to be enabled |
| 40 |
# "by default" |
| 41 |
# - tzdata: while needed to validate fully test_datetime, it makes |
| 42 |
# test_datetime too slow (15-20 min on some buildbots) and so is disabled by |
| 43 |
# default (see bpo-30822). |
| 44 |
RESOURCE_NAMES = ALL_RESOURCES + ('extralargefile', 'tzdata', 'xpickle', 'wantobjects') |
| 45 |
|
| 46 |
|
| 47 |
# Types for types hints |
| 48 |
StrPath = str |
| 49 |
TestName = str |
| 50 |
StrJSON = str |
| 51 |
TestTuple = tuple[TestName, ...] |
| 52 |
TestList = list[TestName] |
| 53 |
# --match and --ignore options: list of patterns |
| 54 |
# ('*' joker character can be used) |
| 55 |
TestFilter = list[tuple[TestName, bool]] |
| 56 |
FilterTuple = tuple[TestName, ...] |
| 57 |
FilterDict = dict[TestName, FilterTuple] |
| 58 |
|
| 59 |
|
| 60 |
def format_duration(seconds: float) -> str: |
| 61 |
ms = math.ceil(seconds * 1e3) |
| 62 |
seconds, ms = divmod(ms, 1000) |
| 63 |
minutes, seconds = divmod(seconds, 60) |
| 64 |
hours, minutes = divmod(minutes, 60) |
| 65 |
|
| 66 |
parts = [] |
| 67 |
if hours: |
| 68 |
parts.append('%s hour' % hours) |
| 69 |
if minutes: |
| 70 |
parts.append('%s min' % minutes) |
| 71 |
if seconds: |
| 72 |
if parts: |
| 73 |
# 2 min 1 sec |
| 74 |
parts.append('%s sec' % seconds) |
| 75 |
else: |
| 76 |
# 1.0 sec |
| 77 |
parts.append('%.1f sec' % (seconds + ms / 1000)) |
| 78 |
if not parts: |
| 79 |
return '%s ms' % ms |
| 80 |
|
| 81 |
parts = parts[:2] |
| 82 |
return ' '.join(parts) |
| 83 |
|
| 84 |
|
| 85 |
def strip_py_suffix(names: list[str] | None) -> None: |
| 86 |
if not names: |
| 87 |
return |
| 88 |
for idx, name in enumerate(names): |
| 89 |
basename, ext = os.path.splitext(name) |
| 90 |
if ext == '.py': |
| 91 |
names[idx] = basename |
| 92 |
|
| 93 |
|
| 94 |
def plural(n: int, singular: str, plural: str | None = None) -> str: |
| 95 |
if n == 1: |
| 96 |
return singular |
| 97 |
elif plural is not None: |
| 98 |
return plural |
| 99 |
else: |
| 100 |
return singular + 's' |
| 101 |
|
| 102 |
|
| 103 |
def count(n: int, word: str) -> str: |
| 104 |
if n == 1: |
| 105 |
return f"{n} {word}" |
| 106 |
else: |
| 107 |
return f"{n} {word}s" |
| 108 |
|
| 109 |
|
| 110 |
def printlist(x, width=70, indent=4, file=None): |
| 111 |
"""Print the elements of iterable x to stdout. |
| 112 |
|
| 113 |
Optional arg width (default 70) is the maximum line length. |
| 114 |
Optional arg indent (default 4) is the number of blanks with which to |
| 115 |
begin each line. |
| 116 |
""" |
| 117 |
|
| 118 |
blanks = ' ' * indent |
| 119 |
# Print the sorted list: 'x' may be a '--random' list or a set() |
| 120 |
print(textwrap.fill(' '.join(str(elt) for elt in sorted(x)), width, |
| 121 |
initial_indent=blanks, subsequent_indent=blanks), |
| 122 |
file=file) |
| 123 |
|
| 124 |
|
| 125 |
def print_warning(msg: str) -> None: |
| 126 |
support.print_warning(msg) |
| 127 |
|
| 128 |
|
| 129 |
orig_unraisablehook: Callable[..., None] | None = None |
| 130 |
|
| 131 |
|
| 132 |
def regrtest_unraisable_hook(unraisable) -> None: |
| 133 |
global orig_unraisablehook |
| 134 |
support.environment_altered = True |
| 135 |
support.print_warning("Unraisable exception") |
| 136 |
old_stderr = sys.stderr |
| 137 |
try: |
| 138 |
support.flush_std_streams() |
| 139 |
sys.stderr = support.print_warning.orig_stderr |
| 140 |
assert orig_unraisablehook is not None, "orig_unraisablehook not set" |
| 141 |
orig_unraisablehook(unraisable) |
| 142 |
sys.stderr.flush() |
| 143 |
finally: |
| 144 |
sys.stderr = old_stderr |
| 145 |
|
| 146 |
|
| 147 |
def setup_unraisable_hook() -> None: |
| 148 |
global orig_unraisablehook |
| 149 |
orig_unraisablehook = sys.unraisablehook |
| 150 |
sys.unraisablehook = regrtest_unraisable_hook |
| 151 |
|
| 152 |
|
| 153 |
orig_threading_excepthook: Callable[..., object] | None = None |
| 154 |
|
| 155 |
|
| 156 |
def regrtest_threading_excepthook(args) -> None: |
| 157 |
global orig_threading_excepthook |
| 158 |
support.environment_altered = True |
| 159 |
support.print_warning(f"Uncaught thread exception: {args.exc_type.__name__}") |
| 160 |
old_stderr = sys.stderr |
| 161 |
try: |
| 162 |
support.flush_std_streams() |
| 163 |
sys.stderr = support.print_warning.orig_stderr |
| 164 |
assert orig_threading_excepthook is not None, "orig_threading_excepthook not set" |
| 165 |
orig_threading_excepthook(args) |
| 166 |
sys.stderr.flush() |
| 167 |
finally: |
| 168 |
sys.stderr = old_stderr |
| 169 |
|
| 170 |
|
| 171 |
def setup_threading_excepthook() -> None: |
| 172 |
global orig_threading_excepthook |
| 173 |
import threading |
| 174 |
orig_threading_excepthook = threading.excepthook |
| 175 |
threading.excepthook = regrtest_threading_excepthook |
| 176 |
|
| 177 |
|
| 178 |
def clear_caches(): |
| 179 |
# Clear the warnings registry, so they can be displayed again |
| 180 |
for mod in sys.modules.values(): |
| 181 |
if hasattr(mod, '__warningregistry__'): |
| 182 |
del mod.__warningregistry__ |
| 183 |
|
| 184 |
# Flush standard output, so that buffered data is sent to the OS and |
| 185 |
# associated Python objects are reclaimed. |
| 186 |
for stream in (sys.stdout, sys.stderr, sys.__stdout__, sys.__stderr__): |
| 187 |
if stream is not None: |
| 188 |
stream.flush() |
| 189 |
|
| 190 |
try: |
| 191 |
re = sys.modules['re'] |
| 192 |
except KeyError: |
| 193 |
pass |
| 194 |
else: |
| 195 |
re.purge() |
| 196 |
|
| 197 |
try: |
| 198 |
_strptime = sys.modules['_strptime'] |
| 199 |
except KeyError: |
| 200 |
pass |
| 201 |
else: |
| 202 |
_strptime._regex_cache.clear() |
| 203 |
|
| 204 |
try: |
| 205 |
urllib_parse = sys.modules['urllib.parse'] |
| 206 |
except KeyError: |
| 207 |
pass |
| 208 |
else: |
| 209 |
urllib_parse.clear_cache() |
| 210 |
|
| 211 |
try: |
| 212 |
urllib_request = sys.modules['urllib.request'] |
| 213 |
except KeyError: |
| 214 |
pass |
| 215 |
else: |
| 216 |
urllib_request.urlcleanup() |
| 217 |
|
| 218 |
try: |
| 219 |
linecache = sys.modules['linecache'] |
| 220 |
except KeyError: |
| 221 |
pass |
| 222 |
else: |
| 223 |
linecache.clearcache() |
| 224 |
|
| 225 |
try: |
| 226 |
mimetypes = sys.modules['mimetypes'] |
| 227 |
except KeyError: |
| 228 |
pass |
| 229 |
else: |
| 230 |
mimetypes._default_mime_types() |
| 231 |
|
| 232 |
try: |
| 233 |
filecmp = sys.modules['filecmp'] |
| 234 |
except KeyError: |
| 235 |
pass |
| 236 |
else: |
| 237 |
filecmp._cache.clear() |
| 238 |
|
| 239 |
try: |
| 240 |
struct = sys.modules['struct'] |
| 241 |
except KeyError: |
| 242 |
pass |
| 243 |
else: |
| 244 |
struct._clearcache() |
| 245 |
|
| 246 |
try: |
| 247 |
doctest = sys.modules['doctest'] |
| 248 |
except KeyError: |
| 249 |
pass |
| 250 |
else: |
| 251 |
doctest.master = None |
| 252 |
|
| 253 |
try: |
| 254 |
ctypes = sys.modules['ctypes'] |
| 255 |
except KeyError: |
| 256 |
pass |
| 257 |
else: |
| 258 |
ctypes._reset_cache() |
| 259 |
|
| 260 |
try: |
| 261 |
typing = sys.modules['typing'] |
| 262 |
except KeyError: |
| 263 |
pass |
| 264 |
else: |
| 265 |
for f in typing._cleanups: |
| 266 |
f() |
| 267 |
|
| 268 |
import inspect |
| 269 |
abs_classes = filter(inspect.isabstract, typing.__dict__.values()) |
| 270 |
for abc in abs_classes: |
| 271 |
for obj in abc.__subclasses__() + [abc]: |
| 272 |
obj._abc_caches_clear() |
| 273 |
|
| 274 |
try: |
| 275 |
fractions = sys.modules['fractions'] |
| 276 |
except KeyError: |
| 277 |
pass |
| 278 |
else: |
| 279 |
fractions._hash_algorithm.cache_clear() |
| 280 |
|
| 281 |
try: |
| 282 |
inspect = sys.modules['inspect'] |
| 283 |
except KeyError: |
| 284 |
pass |
| 285 |
else: |
| 286 |
inspect._shadowed_dict_from_weakref_mro_tuple.cache_clear() |
| 287 |
inspect._filesbymodname.clear() |
| 288 |
inspect.modulesbyfile.clear() |
| 289 |
|
| 290 |
try: |
| 291 |
importlib_metadata = sys.modules['importlib.metadata'] |
| 292 |
except KeyError: |
| 293 |
pass |
| 294 |
else: |
| 295 |
importlib_metadata.FastPath.__new__.cache_clear() |
| 296 |
|
| 297 |
|
| 298 |
def get_build_info(): |
| 299 |
# Get most important configure and build options as a list of strings. |
| 300 |
# Example: ['debug', 'ASAN+MSAN'] or ['release', 'LTO+PGO']. |
| 301 |
|
| 302 |
config_args = sysconfig.get_config_var('CONFIG_ARGS') or '' |
| 303 |
cflags = sysconfig.get_config_var('PY_CFLAGS') or '' |
| 304 |
cflags += ' ' + (sysconfig.get_config_var('PY_CFLAGS_NODIST') or '') |
| 305 |
ldflags_nodist = sysconfig.get_config_var('PY_LDFLAGS_NODIST') or '' |
| 306 |
|
| 307 |
build = [] |
| 308 |
|
| 309 |
# --disable-gil |
| 310 |
if sysconfig.get_config_var('Py_GIL_DISABLED'): |
| 311 |
if not sys.flags.ignore_environment: |
| 312 |
PYTHON_GIL = os.environ.get('PYTHON_GIL', None) |
| 313 |
if PYTHON_GIL: |
| 314 |
PYTHON_GIL = (PYTHON_GIL == '1') |
| 315 |
else: |
| 316 |
PYTHON_GIL = None |
| 317 |
|
| 318 |
free_threading = "free_threading" |
| 319 |
if PYTHON_GIL is not None: |
| 320 |
free_threading = f"{free_threading} GIL={int(PYTHON_GIL)}" |
| 321 |
build.append(free_threading) |
| 322 |
|
| 323 |
if hasattr(sys, 'gettotalrefcount'): |
| 324 |
# --with-pydebug |
| 325 |
build.append('debug') |
| 326 |
|
| 327 |
if '-DNDEBUG' in cflags: |
| 328 |
build.append('without_assert') |
| 329 |
else: |
| 330 |
build.append('release') |
| 331 |
|
| 332 |
if '--with-assertions' in config_args: |
| 333 |
build.append('with_assert') |
| 334 |
elif '-DNDEBUG' not in cflags: |
| 335 |
build.append('with_assert') |
| 336 |
|
| 337 |
# --enable-experimental-jit |
| 338 |
if sys._jit.is_available(): |
| 339 |
if sys._jit.is_enabled(): |
| 340 |
build.append("JIT") |
| 341 |
else: |
| 342 |
build.append("JIT (disabled)") |
| 343 |
|
| 344 |
# --enable-framework=name |
| 345 |
framework = sysconfig.get_config_var('PYTHONFRAMEWORK') |
| 346 |
if framework: |
| 347 |
build.append(f'framework={framework}') |
| 348 |
|
| 349 |
# --enable-shared |
| 350 |
shared = int(sysconfig.get_config_var('PY_ENABLE_SHARED') or '0') |
| 351 |
if shared: |
| 352 |
build.append('shared') |
| 353 |
|
| 354 |
# --with-lto |
| 355 |
optimizations = [] |
| 356 |
if '-flto=thin' in ldflags_nodist: |
| 357 |
optimizations.append('ThinLTO') |
| 358 |
elif '-flto' in ldflags_nodist: |
| 359 |
optimizations.append('LTO') |
| 360 |
|
| 361 |
if support.check_cflags_pgo(): |
| 362 |
# PGO (--enable-optimizations) |
| 363 |
optimizations.append('PGO') |
| 364 |
|
| 365 |
if support.check_bolt_optimized(): |
| 366 |
# BOLT (--enable-bolt) |
| 367 |
optimizations.append('BOLT') |
| 368 |
|
| 369 |
if optimizations: |
| 370 |
build.append('+'.join(optimizations)) |
| 371 |
|
| 372 |
# --with-address-sanitizer |
| 373 |
sanitizers = [] |
| 374 |
if support.check_sanitizer(address=True): |
| 375 |
sanitizers.append("ASAN") |
| 376 |
# --with-memory-sanitizer |
| 377 |
if support.check_sanitizer(memory=True): |
| 378 |
sanitizers.append("MSAN") |
| 379 |
# --with-undefined-behavior-sanitizer |
| 380 |
if support.check_sanitizer(ub=True): |
| 381 |
sanitizers.append("UBSAN") |
| 382 |
# --with-thread-sanitizer |
| 383 |
if support.check_sanitizer(thread=True): |
| 384 |
sanitizers.append("TSAN") |
| 385 |
if sanitizers: |
| 386 |
build.append('+'.join(sanitizers)) |
| 387 |
|
| 388 |
# --with-trace-refs |
| 389 |
if hasattr(sys, 'getobjects'): |
| 390 |
build.append("TraceRefs") |
| 391 |
# --enable-pystats |
| 392 |
if hasattr(sys, '_stats_on'): |
| 393 |
build.append("pystats") |
| 394 |
# --with-valgrind |
| 395 |
if sysconfig.get_config_var('WITH_VALGRIND'): |
| 396 |
build.append("valgrind") |
| 397 |
# --with-dtrace |
| 398 |
if sysconfig.get_config_var('WITH_DTRACE'): |
| 399 |
build.append("dtrace") |
| 400 |
|
| 401 |
return build |
| 402 |
|
| 403 |
|
| 404 |
def get_temp_dir(tmp_dir: StrPath | None = None) -> StrPath: |
| 405 |
if tmp_dir: |
| 406 |
tmp_dir = os.path.expanduser(tmp_dir) |
| 407 |
else: |
| 408 |
# When tests are run from the Python build directory, it is best practice |
| 409 |
# to keep the test files in a subfolder. This eases the cleanup of leftover |
| 410 |
# files using the "make distclean" command. |
| 411 |
if sysconfig.is_python_build(): |
| 412 |
if not support.is_wasi: |
| 413 |
tmp_dir = sysconfig.get_config_var('abs_builddir') |
| 414 |
if tmp_dir is None: |
| 415 |
tmp_dir = sysconfig.get_config_var('abs_srcdir') |
| 416 |
if not tmp_dir: |
| 417 |
# gh-74470: On Windows, only srcdir is available. Using |
| 418 |
# abs_builddir mostly matters on UNIX when building |
| 419 |
# Python out of the source tree, especially when the |
| 420 |
# source tree is read only. |
| 421 |
tmp_dir = sysconfig.get_config_var('srcdir') |
| 422 |
if not tmp_dir: |
| 423 |
raise RuntimeError( |
| 424 |
"Could not determine the correct value for tmp_dir" |
| 425 |
) |
| 426 |
tmp_dir = os.path.join(tmp_dir, 'build') |
| 427 |
else: |
| 428 |
# WASI platform |
| 429 |
tmp_dir = sysconfig.get_config_var('projectbase') |
| 430 |
if not tmp_dir: |
| 431 |
raise RuntimeError( |
| 432 |
"sysconfig.get_config_var('projectbase') " |
| 433 |
f"unexpectedly returned {tmp_dir!r} on WASI" |
| 434 |
) |
| 435 |
tmp_dir = os.path.join(tmp_dir, 'build') |
| 436 |
else: |
| 437 |
tmp_dir = tempfile.gettempdir() |
| 438 |
|
| 439 |
return os.path.abspath(tmp_dir) |
| 440 |
|
| 441 |
|
| 442 |
def get_work_dir(parent_dir: StrPath, worker: bool = False) -> StrPath: |
| 443 |
# Define a writable temp dir that will be used as cwd while running |
| 444 |
# the tests. The name of the dir includes the pid to allow parallel |
| 445 |
# testing (see the -j option). |
| 446 |
# Emscripten and WASI have stubbed getpid(), Emscripten has only |
| 447 |
# millisecond clock resolution. Use randint() instead. |
| 448 |
if support.is_emscripten or support.is_wasi: |
| 449 |
nounce = random.randint(0, 1_000_000) |
| 450 |
else: |
| 451 |
nounce = os.getpid() |
| 452 |
|
| 453 |
if worker: |
| 454 |
work_dir = WORK_DIR_PREFIX + str(nounce) |
| 455 |
else: |
| 456 |
work_dir = WORKER_WORK_DIR_PREFIX + str(nounce) |
| 457 |
work_dir += os_helper.FS_NONASCII |
| 458 |
work_dir = os.path.join(parent_dir, work_dir) |
| 459 |
return work_dir |
| 460 |
|
| 461 |
|
| 462 |
@contextlib.contextmanager |
| 463 |
def exit_timeout(): |
| 464 |
try: |
| 465 |
yield |
| 466 |
except SystemExit as exc: |
| 467 |
# bpo-38203: Python can hang at exit in Py_Finalize(), especially |
| 468 |
# on threading._shutdown() call: put a timeout |
| 469 |
if threading_helper.can_start_thread: |
| 470 |
faulthandler.dump_traceback_later(EXIT_TIMEOUT, exit=True) |
| 471 |
sys.exit(exc.code) |
| 472 |
|
| 473 |
|
| 474 |
def remove_testfn(test_name: TestName, verbose: int) -> None: |
| 475 |
# Try to clean up os_helper.TESTFN if left behind. |
| 476 |
# |
| 477 |
# While tests shouldn't leave any files or directories behind, when a test |
| 478 |
# fails that can be tedious for it to arrange. The consequences can be |
| 479 |
# especially nasty on Windows, since if a test leaves a file open, it |
| 480 |
# cannot be deleted by name (while there's nothing we can do about that |
| 481 |
# here either, we can display the name of the offending test, which is a |
| 482 |
# real help). |
| 483 |
name = os_helper.TESTFN |
| 484 |
if not os.path.exists(name): |
| 485 |
return |
| 486 |
|
| 487 |
nuker: Callable[[str], None] |
| 488 |
if os.path.isdir(name): |
| 489 |
import shutil |
| 490 |
kind, nuker = "directory", shutil.rmtree |
| 491 |
elif os.path.isfile(name): |
| 492 |
kind, nuker = "file", os.unlink |
| 493 |
else: |
| 494 |
raise RuntimeError(f"os.path says {name!r} exists but is neither " |
| 495 |
f"directory nor file") |
| 496 |
|
| 497 |
if verbose: |
| 498 |
print_warning(f"{test_name} left behind {kind} {name!r}") |
| 499 |
support.environment_altered = True |
| 500 |
|
| 501 |
try: |
| 502 |
import stat |
| 503 |
# fix possible permissions problems that might prevent cleanup |
| 504 |
os.chmod(name, stat.S_IRWXU | stat.S_IRWXG | stat.S_IRWXO) |
| 505 |
nuker(name) |
| 506 |
except Exception as exc: |
| 507 |
print_warning(f"{test_name} left behind {kind} {name!r} " |
| 508 |
f"and it couldn't be removed: {exc}") |
| 509 |
|
| 510 |
|
| 511 |
def abs_module_name(test_name: TestName, test_dir: StrPath | None) -> TestName: |
| 512 |
if test_name.startswith('test.') or test_dir: |
| 513 |
return test_name |
| 514 |
else: |
| 515 |
# Import it from the test package |
| 516 |
return 'test.' + test_name |
| 517 |
|
| 518 |
|
| 519 |
# gh-90681: When rerunning tests, we might need to rerun the whole |
| 520 |
# class or module suite if some its life-cycle hooks fail. |
| 521 |
# Test level hooks are not affected. |
| 522 |
_TEST_LIFECYCLE_HOOKS = frozenset(( |
| 523 |
'setUpClass', 'tearDownClass', |
| 524 |
'setUpModule', 'tearDownModule', |
| 525 |
)) |
| 526 |
|
| 527 |
def normalize_test_name(test_full_name: str, *, |
| 528 |
is_error: bool = False) -> str | None: |
| 529 |
short_name = test_full_name.split(" ")[0] |
| 530 |
if is_error and short_name in _TEST_LIFECYCLE_HOOKS: |
| 531 |
if test_full_name.startswith(('setUpModule (', 'tearDownModule (')): |
| 532 |
# if setUpModule() or tearDownModule() failed, don't filter |
| 533 |
# tests with the test file name, don't use filters. |
| 534 |
return None |
| 535 |
|
| 536 |
# This means that we have a failure in a life-cycle hook, |
| 537 |
# we need to rerun the whole module or class suite. |
| 538 |
# Basically the error looks like this: |
| 539 |
# ERROR: setUpClass (test.test_reg_ex.RegTest) |
| 540 |
# or |
| 541 |
# ERROR: setUpModule (test.test_reg_ex) |
| 542 |
# So, we need to parse the class / module name. |
| 543 |
lpar = test_full_name.index('(') |
| 544 |
rpar = test_full_name.index(')') |
| 545 |
return test_full_name[lpar + 1: rpar].split('.')[-1] |
| 546 |
return short_name |
| 547 |
|
| 548 |
|
| 549 |
def adjust_rlimit_nofile() -> None: |
| 550 |
""" |
| 551 |
On macOS the default fd limit (RLIMIT_NOFILE) is sometimes too low (256) |
| 552 |
for our test suite to succeed. Raise it to something more reasonable. 1024 |
| 553 |
is a common Linux default. |
| 554 |
""" |
| 555 |
try: |
| 556 |
import resource |
| 557 |
except ImportError: |
| 558 |
return |
| 559 |
|
| 560 |
fd_limit, max_fds = resource.getrlimit(resource.RLIMIT_NOFILE) |
| 561 |
|
| 562 |
desired_fds = 1024 |
| 563 |
|
| 564 |
if fd_limit < desired_fds and fd_limit < max_fds: |
| 565 |
new_fd_limit = min(desired_fds, max_fds) |
| 566 |
try: |
| 567 |
resource.setrlimit(resource.RLIMIT_NOFILE, |
| 568 |
(new_fd_limit, max_fds)) |
| 569 |
print(f"Raised RLIMIT_NOFILE: {fd_limit} -> {new_fd_limit}") |
| 570 |
except (ValueError, OSError) as err: |
| 571 |
print_warning(f"Unable to raise RLIMIT_NOFILE from {fd_limit} to " |
| 572 |
f"{new_fd_limit}: {err}.") |
| 573 |
|
| 574 |
|
| 575 |
def get_host_runner() -> str: |
| 576 |
if (hostrunner := os.environ.get("_PYTHON_HOSTRUNNER")) is None: |
| 577 |
hostrunner = sysconfig.get_config_var("HOSTRUNNER") |
| 578 |
return hostrunner |
| 579 |
|
| 580 |
|
| 581 |
def is_cross_compiled() -> bool: |
| 582 |
return ('_PYTHON_HOST_PLATFORM' in os.environ) |
| 583 |
|
| 584 |
|
| 585 |
def format_resources(use_resources: dict[str, str | None]) -> str: |
| 586 |
all_resources = set(ALL_RESOURCES) |
| 587 |
|
| 588 |
values = [] |
| 589 |
for name in sorted(use_resources): |
| 590 |
if use_resources[name] is not None: |
| 591 |
values.append(f'{name}={use_resources[name]}') |
| 592 |
|
| 593 |
# Express resources relative to "all" |
| 594 |
relative_all = ['all'] |
| 595 |
for name in sorted(all_resources - set(use_resources)): |
| 596 |
relative_all.append(f'-{name}') |
| 597 |
for name in sorted(set(use_resources) - all_resources): |
| 598 |
if use_resources[name] is None: |
| 599 |
relative_all.append(name) |
| 600 |
all_text = ','.join(relative_all + values) |
| 601 |
all_text = f"resources: {all_text}" |
| 602 |
|
| 603 |
# List of enabled resources |
| 604 |
resources = [] |
| 605 |
for name in sorted(use_resources): |
| 606 |
if use_resources[name] is None: |
| 607 |
resources.append(name) |
| 608 |
text = ','.join(resources + values) |
| 609 |
text = f"resources ({len(use_resources)}): {text}" |
| 610 |
|
| 611 |
# Pick the shortest string (prefer relative to all if lengths are equal) |
| 612 |
if len(all_text) <= len(text): |
| 613 |
return all_text |
| 614 |
else: |
| 615 |
return text |
| 616 |
|
| 617 |
|
| 618 |
def display_header(use_resources: dict[str, str | None], |
| 619 |
python_cmd: tuple[str, ...] | None) -> None: |
| 620 |
# Print basic platform information |
| 621 |
print("==", platform.python_implementation(), *sys.version.split()) |
| 622 |
print("==", platform.platform(aliased=True), |
| 623 |
"%s-endian" % sys.byteorder) |
| 624 |
print("== Python build:", ' '.join(get_build_info())) |
| 625 |
print("== cwd:", os.getcwd()) |
| 626 |
|
| 627 |
cpu_count: object = os.cpu_count() |
| 628 |
if cpu_count: |
| 629 |
# The function is new in Python 3.13; mypy doesn't know about it yet: |
| 630 |
process_cpu_count = os.process_cpu_count() # type: ignore[attr-defined] |
| 631 |
if process_cpu_count and process_cpu_count != cpu_count: |
| 632 |
cpu_count = f"{process_cpu_count} (process) / {cpu_count} (system)" |
| 633 |
print("== CPU count:", cpu_count) |
| 634 |
print("== encodings: locale=%s FS=%s" |
| 635 |
% (locale.getencoding(), sys.getfilesystemencoding())) |
| 636 |
|
| 637 |
if use_resources: |
| 638 |
text = format_resources(use_resources) |
| 639 |
print(f"== {text}") |
| 640 |
else: |
| 641 |
print("== resources: all test resources are disabled, " |
| 642 |
"use -u option to unskip tests") |
| 643 |
|
| 644 |
cross_compile = is_cross_compiled() |
| 645 |
if cross_compile: |
| 646 |
print("== cross compiled: Yes") |
| 647 |
if python_cmd: |
| 648 |
cmd = shlex.join(python_cmd) |
| 649 |
print(f"== host python: {cmd}") |
| 650 |
|
| 651 |
get_cmd = [*python_cmd, '-m', 'platform'] |
| 652 |
proc = subprocess.run( |
| 653 |
get_cmd, |
| 654 |
stdout=subprocess.PIPE, |
| 655 |
text=True, |
| 656 |
cwd=os_helper.SAVEDCWD) |
| 657 |
stdout = proc.stdout.replace('\n', ' ').strip() |
| 658 |
if stdout: |
| 659 |
print(f"== host platform: {stdout}") |
| 660 |
elif proc.returncode: |
| 661 |
print(f"== host platform: <command failed with exit code {proc.returncode}>") |
| 662 |
else: |
| 663 |
hostrunner = get_host_runner() |
| 664 |
if hostrunner: |
| 665 |
print(f"== host runner: {hostrunner}") |
| 666 |
|
| 667 |
# This makes it easier to remember what to set in your local |
| 668 |
# environment when trying to reproduce a sanitizer failure. |
| 669 |
asan = support.check_sanitizer(address=True) |
| 670 |
msan = support.check_sanitizer(memory=True) |
| 671 |
ubsan = support.check_sanitizer(ub=True) |
| 672 |
tsan = support.check_sanitizer(thread=True) |
| 673 |
sanitizers = [] |
| 674 |
if asan: |
| 675 |
sanitizers.append("address") |
| 676 |
if msan: |
| 677 |
sanitizers.append("memory") |
| 678 |
if ubsan: |
| 679 |
sanitizers.append("undefined behavior") |
| 680 |
if tsan: |
| 681 |
sanitizers.append("thread") |
| 682 |
if sanitizers: |
| 683 |
print(f"== sanitizers: {', '.join(sanitizers)}") |
| 684 |
for sanitizer, env_var in ( |
| 685 |
(asan, "ASAN_OPTIONS"), |
| 686 |
(msan, "MSAN_OPTIONS"), |
| 687 |
(ubsan, "UBSAN_OPTIONS"), |
| 688 |
(tsan, "TSAN_OPTIONS"), |
| 689 |
): |
| 690 |
options= os.environ.get(env_var) |
| 691 |
if sanitizer and options is not None: |
| 692 |
print(f"== {env_var}={options!r}") |
| 693 |
|
| 694 |
print(flush=True) |
| 695 |
|
| 696 |
|
| 697 |
def cleanup_temp_dir(tmp_dir: StrPath) -> None: |
| 698 |
import glob |
| 699 |
|
| 700 |
path = os.path.join(glob.escape(tmp_dir), TMP_PREFIX + '*') |
| 701 |
print("Cleanup %s directory" % tmp_dir) |
| 702 |
for name in glob.glob(path): |
| 703 |
if os.path.isdir(name): |
| 704 |
print("Remove directory: %s" % name) |
| 705 |
os_helper.rmtree(name) |
| 706 |
else: |
| 707 |
print("Remove file: %s" % name) |
| 708 |
os_helper.unlink(name) |
| 709 |
|
| 710 |
|
| 711 |
ILLEGAL_XML_CHARS_RE = re.compile( |
| 712 |
'[' |
| 713 |
# Control characters; newline (\x0A and \x0D) and TAB (\x09) are legal |
| 714 |
'\x00-\x08\x0B\x0C\x0E-\x1F' |
| 715 |
# Surrogate characters |
| 716 |
'\uD800-\uDFFF' |
| 717 |
# Special Unicode characters |
| 718 |
'\uFFFE' |
| 719 |
'\uFFFF' |
| 720 |
# Match multiple sequential invalid characters for better efficiency |
| 721 |
']+') |
| 722 |
|
| 723 |
def _sanitize_xml_replace(regs): |
| 724 |
text = regs[0] |
| 725 |
return ''.join(f'\\x{ord(ch):02x}' if ch <= '\xff' else ascii(ch)[1:-1] |
| 726 |
for ch in text) |
| 727 |
|
| 728 |
def sanitize_xml(text: str) -> str: |
| 729 |
return ILLEGAL_XML_CHARS_RE.sub(_sanitize_xml_replace, text) |