Debian Codegraph @

Lib/test/libregrtest/run_workers.py

raw
1 import contextlib
2 import dataclasses
3 import faulthandler
4 import os.path
5 import queue
6 import signal
7 import subprocess
8 import sys
9 import tempfile
10 import threading
11 import time
12 import traceback
13 from typing import Any, Literal, TextIO
14
15 from test import support
16 from test.support import os_helper, MS_WINDOWS
17
18 from .logger import Logger
19 from .result import TestResult, State
20 from .results import TestResults
21 from .runtests import RunTests, WorkerRunTests, JsonFile, JsonFileType
22 from .single import PROGRESS_MIN_TIME
23 from .utils import (
24 StrPath, TestName,
25 format_duration, print_warning, count, plural)
26 from .worker import create_worker_process, USE_PROCESS_GROUP
27
28 if MS_WINDOWS:
29 import locale
30 import msvcrt
31
32
33
34 # Display the running tests if nothing happened last N seconds
35 PROGRESS_UPDATE = 30.0 # seconds
36 assert PROGRESS_UPDATE >= PROGRESS_MIN_TIME
37
38 # Kill the main process after 5 minutes. It is supposed to write an update
39 # every PROGRESS_UPDATE seconds. Tolerate 5 minutes for Python slowest
40 # buildbot workers.
41 MAIN_PROCESS_TIMEOUT = 5 * 60.0
42 assert MAIN_PROCESS_TIMEOUT >= PROGRESS_UPDATE
43
44 # Time to wait until a worker completes: should be immediate
45 WAIT_COMPLETED_TIMEOUT = 30.0 # seconds
46
47 # Time to wait a killed process (in seconds)
48 WAIT_KILLED_TIMEOUT = 60.0
49
50
51 # We do not use a generator so multiple threads can call next().
52 class MultiprocessIterator:
53
54 """A thread-safe iterator over tests for multiprocess mode."""
55
56 def __init__(self, tests_iter):
57 self.lock = threading.Lock()
58 self.tests_iter = tests_iter
59
60 def __iter__(self):
61 return self
62
63 def __next__(self):
64 with self.lock:
65 if self.tests_iter is None:
66 raise StopIteration
67 return next(self.tests_iter)
68
69 def stop(self):
70 with self.lock:
71 self.tests_iter = None
72
73
74 @dataclasses.dataclass(slots=True, frozen=True)
75 class MultiprocessResult:
76 result: TestResult
77 # bpo-45410: stderr is written into stdout to keep messages order
78 worker_stdout: str | None = None
79 err_msg: str | None = None
80
81
82 class WorkerThreadExited:
83 """Indicates that a worker thread has exited"""
84
85 ExcStr = str
86 QueueOutput = tuple[Literal[False], MultiprocessResult] | tuple[Literal[True], ExcStr]
87 QueueContent = QueueOutput | WorkerThreadExited
88
89
90 class ExitThread(Exception):
91 pass
92
93
94 class WorkerError(Exception):
95 def __init__(self,
96 test_name: TestName,
97 err_msg: str | None,
98 stdout: str | None,
99 state: str):
100 result = TestResult(test_name, state=state)
101 self.mp_result = MultiprocessResult(result, stdout, err_msg)
102 super().__init__()
103
104
105 _NOT_RUNNING = "<not running>"
106
107
108 class WorkerThread(threading.Thread):
109 def __init__(self, worker_id: int, runner: "RunWorkers") -> None:
110 super().__init__()
111 self.worker_id = worker_id
112 self.runtests = runner.runtests
113 self.pending = runner.pending
114 self.output = runner.output
115 self.timeout = runner.worker_timeout
116 self.log = runner.log
117 self.test_name = _NOT_RUNNING
118 self.start_time = time.monotonic()
119 self._popen: subprocess.Popen[str] | None = None
120 self._killed = False
121 self._stopped = False
122
123 def __repr__(self) -> str:
124 info = [f'WorkerThread #{self.worker_id}']
125 if self.is_alive():
126 info.append("running")
127 else:
128 info.append('stopped')
129 test = self.test_name
130 if test:
131 info.append(f'test={test}')
132 popen = self._popen
133 if popen is not None:
134 dt = time.monotonic() - self.start_time
135 info.extend((f'pid={popen.pid}',
136 f'time={format_duration(dt)}'))
137 return '<%s>' % ' '.join(info)
138
139 def _kill(self) -> None:
140 popen = self._popen
141 if popen is None:
142 return
143
144 if self._killed:
145 return
146 self._killed = True
147
148 use_killpg = USE_PROCESS_GROUP
149 if use_killpg:
150 parent_sid = os.getsid(0)
151 sid = os.getsid(popen.pid)
152 use_killpg = (sid != parent_sid)
153
154 if use_killpg:
155 what = f"{self} process group"
156 else:
157 what = f"{self} process"
158
159 print(f"Kill {what}", file=sys.stderr, flush=True)
160 try:
161 if use_killpg:
162 os.killpg(popen.pid, signal.SIGKILL)
163 else:
164 popen.kill()
165 except ProcessLookupError:
166 # popen.kill(): the process completed, the WorkerThread thread
167 # read its exit status, but Popen.send_signal() read the returncode
168 # just before Popen.wait() set returncode.
169 pass
170 except OSError as exc:
171 print_warning(f"Failed to kill {what}: {exc!r}")
172
173 def stop(self) -> None:
174 # Method called from a different thread to stop this thread
175 self._stopped = True
176 self._kill()
177
178 def _run_process(self, runtests: WorkerRunTests, output_fd: int,
179 tmp_dir: StrPath | None = None) -> int | None:
180 popen = create_worker_process(runtests, output_fd, tmp_dir)
181 self._popen = popen
182 self._killed = False
183
184 try:
185 if self._stopped:
186 # If kill() has been called before self._popen is set,
187 # self._popen is still running. Call again kill()
188 # to ensure that the process is killed.
189 self._kill()
190 raise ExitThread
191
192 try:
193 # gh-94026: stdout+stderr are written to tempfile
194 retcode = popen.wait(timeout=self.timeout)
195 assert retcode is not None
196 return retcode
197 except subprocess.TimeoutExpired:
198 if self._stopped:
199 # kill() has been called: communicate() fails on reading
200 # closed stdout
201 raise ExitThread
202
203 # On timeout, kill the process
204 self._kill()
205
206 # None means TIMEOUT for the caller
207 retcode = None
208 # bpo-38207: Don't attempt to call communicate() again: on it
209 # can hang until all child processes using stdout
210 # pipes completes.
211 except OSError:
212 if self._stopped:
213 # kill() has been called: communicate() fails
214 # on reading closed stdout
215 raise ExitThread
216 raise
217 return None
218 except:
219 self._kill()
220 raise
221 finally:
222 self._wait_completed()
223 self._popen = None
224
225 def create_stdout(self, stack: contextlib.ExitStack) -> TextIO:
226 """Create stdout temporary file (file descriptor)."""
227
228 if MS_WINDOWS:
229 # gh-95027: When stdout is not a TTY, Python uses the ANSI code
230 # page for the sys.stdout encoding. If the main process runs in a
231 # terminal, sys.stdout uses WindowsConsoleIO with UTF-8 encoding.
232 encoding = locale.getencoding()
233 else:
234 encoding = sys.stdout.encoding
235
236 # gh-94026: Write stdout+stderr to a tempfile as workaround for
237 # non-blocking pipes on Emscripten with NodeJS.
238 # gh-109425: Use "backslashreplace" error handler: log corrupted
239 # stdout+stderr, instead of failing with a UnicodeDecodeError and not
240 # logging stdout+stderr at all.
241 stdout_file = tempfile.TemporaryFile('w+',
242 encoding=encoding,
243 errors='backslashreplace')
244 stack.enter_context(stdout_file)
245 return stdout_file
246
247 def create_json_file(self, stack: contextlib.ExitStack) -> tuple[JsonFile, TextIO | None]:
248 """Create JSON file."""
249
250 json_file_use_stdout = self.runtests.json_file_use_stdout()
251 if json_file_use_stdout:
252 json_file = JsonFile(None, JsonFileType.STDOUT)
253 json_tmpfile = None
254 else:
255 json_tmpfile = tempfile.TemporaryFile('w+', encoding='utf8')
256 stack.enter_context(json_tmpfile)
257
258 json_fd = json_tmpfile.fileno()
259 if MS_WINDOWS:
260 # The msvcrt module is only available on Windows;
261 # we run mypy with `--platform=linux` in CI
262 json_handle: int = msvcrt.get_osfhandle(json_fd) # type: ignore[attr-defined]
263 json_file = JsonFile(json_handle,
264 JsonFileType.WINDOWS_HANDLE)
265 else:
266 json_file = JsonFile(json_fd, JsonFileType.UNIX_FD)
267 return (json_file, json_tmpfile)
268
269 def create_worker_runtests(self, test_name: TestName, json_file: JsonFile) -> WorkerRunTests:
270 tests = (test_name,)
271 if self.runtests.rerun:
272 match_tests = self.runtests.get_match_tests(test_name)
273 else:
274 match_tests = None
275
276 kwargs: dict[str, Any] = {}
277 if match_tests:
278 kwargs['match_tests'] = [(test, True) for test in match_tests]
279 if self.runtests.output_on_failure:
280 kwargs['verbose'] = True
281 kwargs['output_on_failure'] = False
282 return self.runtests.create_worker_runtests(
283 tests=tests,
284 json_file=json_file,
285 **kwargs)
286
287 def run_tmp_files(self, worker_runtests: WorkerRunTests,
288 stdout_fd: int) -> tuple[int | None, list[StrPath]]:
289 # gh-93353: Check for leaked temporary files in the parent process,
290 # since the deletion of temporary files can happen late during
291 # Python finalization: too late for libregrtest.
292 if not support.is_wasi:
293 # Don't check for leaked temporary files and directories if Python is
294 # run on WASI. WASI doesn't pass environment variables like TMPDIR to
295 # worker processes.
296 tmp_dir = tempfile.mkdtemp(prefix="test_python_")
297 tmp_dir = os.path.abspath(tmp_dir)
298 try:
299 retcode = self._run_process(worker_runtests,
300 stdout_fd, tmp_dir)
301 finally:
302 tmp_files = os.listdir(tmp_dir)
303 os_helper.rmtree(tmp_dir)
304 else:
305 retcode = self._run_process(worker_runtests, stdout_fd)
306 tmp_files = []
307
308 return (retcode, tmp_files)
309
310 def read_stdout(self, stdout_file: TextIO) -> str:
311 stdout_file.seek(0)
312 try:
313 return stdout_file.read().strip()
314 except Exception as exc:
315 # gh-101634: Catch UnicodeDecodeError if stdout cannot be
316 # decoded from encoding
317 raise WorkerError(self.test_name,
318 f"Cannot read process stdout: {exc}",
319 stdout=None,
320 state=State.WORKER_BUG)
321
322 def read_json(self, json_file: JsonFile, json_tmpfile: TextIO | None,
323 stdout: str) -> tuple[TestResult, str]:
324 try:
325 if json_tmpfile is not None:
326 json_tmpfile.seek(0)
327 worker_json = json_tmpfile.read()
328 elif json_file.file_type == JsonFileType.STDOUT:
329 stdout, _, worker_json = stdout.rpartition("\n")
330 stdout = stdout.rstrip()
331 else:
332 with json_file.open(encoding='utf8') as json_fp:
333 worker_json = json_fp.read()
334 except Exception as exc:
335 # gh-101634: Catch UnicodeDecodeError if stdout cannot be
336 # decoded from encoding
337 err_msg = f"Failed to read worker process JSON: {exc}"
338 raise WorkerError(self.test_name, err_msg, stdout,
339 state=State.WORKER_BUG)
340
341 if not worker_json:
342 raise WorkerError(self.test_name, "empty JSON", stdout,
343 state=State.WORKER_BUG)
344
345 try:
346 result = TestResult.from_json(worker_json)
347 except Exception as exc:
348 # gh-101634: Catch UnicodeDecodeError if stdout cannot be
349 # decoded from encoding
350 err_msg = f"Failed to parse worker process JSON: {exc}"
351 raise WorkerError(self.test_name, err_msg, stdout,
352 state=State.WORKER_BUG)
353
354 return (result, stdout)
355
356 def _runtest(self, test_name: TestName) -> MultiprocessResult:
357 with contextlib.ExitStack() as stack:
358 stdout_file = self.create_stdout(stack)
359 json_file, json_tmpfile = self.create_json_file(stack)
360 worker_runtests = self.create_worker_runtests(test_name, json_file)
361
362 retcode: str | int | None
363 retcode, tmp_files = self.run_tmp_files(worker_runtests,
364 stdout_file.fileno())
365
366 stdout = self.read_stdout(stdout_file)
367
368 if retcode is None:
369 raise WorkerError(self.test_name, stdout=stdout,
370 err_msg=None,
371 state=State.TIMEOUT)
372 if retcode != 0:
373 name = support.get_signal_name(retcode)
374 if name:
375 retcode = f"{retcode} ({name})"
376 raise WorkerError(self.test_name, f"Exit code {retcode}", stdout,
377 state=State.WORKER_FAILED)
378
379 result, stdout = self.read_json(json_file, json_tmpfile, stdout)
380
381 if tmp_files:
382 msg = (f'\n\n'
383 f'Warning -- {test_name} leaked temporary files '
384 f'({len(tmp_files)}): {", ".join(sorted(tmp_files))}')
385 stdout += msg
386 result.set_env_changed()
387
388 return MultiprocessResult(result, stdout)
389
390 def run(self) -> None:
391 fail_fast = self.runtests.fail_fast
392 fail_env_changed = self.runtests.fail_env_changed
393 try:
394 while not self._stopped:
395 try:
396 test_name = next(self.pending)
397 except StopIteration:
398 break
399
400 self.start_time = time.monotonic()
401 self.test_name = test_name
402 try:
403 mp_result = self._runtest(test_name)
404 except WorkerError as exc:
405 mp_result = exc.mp_result
406 finally:
407 self.test_name = _NOT_RUNNING
408 mp_result.result.duration = time.monotonic() - self.start_time
409 self.output.put((False, mp_result))
410
411 if mp_result.result.must_stop(fail_fast, fail_env_changed):
412 break
413 except ExitThread:
414 pass
415 except BaseException:
416 self.output.put((True, traceback.format_exc()))
417 finally:
418 self.output.put(WorkerThreadExited())
419
420 def _wait_completed(self) -> None:
421 popen = self._popen
422 # only needed for mypy:
423 if popen is None:
424 raise ValueError("Should never access `._popen` before calling `.run()`")
425
426 try:
427 popen.wait(WAIT_COMPLETED_TIMEOUT)
428 except (subprocess.TimeoutExpired, OSError) as exc:
429 print_warning(f"Failed to wait for {self} completion "
430 f"(timeout={format_duration(WAIT_COMPLETED_TIMEOUT)}): "
431 f"{exc!r}")
432
433 def wait_stopped(self, start_time: float) -> None:
434 # bpo-38207: RunWorkers.stop_workers() called self.stop()
435 # which killed the process. Sometimes, killing the process from the
436 # main thread does not interrupt popen.communicate() in
437 # WorkerThread thread. This loop with a timeout is a workaround
438 # for that.
439 #
440 # Moreover, if this method fails to join the thread, it is likely
441 # that Python will hang at exit while calling threading._shutdown()
442 # which tries again to join the blocked thread. Regrtest.main()
443 # uses EXIT_TIMEOUT to workaround this second bug.
444 while True:
445 # Write a message every second
446 self.join(1.0)
447 if not self.is_alive():
448 break
449 dt = time.monotonic() - start_time
450 self.log(f"Waiting for {self} thread for {format_duration(dt)}")
451 if dt > WAIT_KILLED_TIMEOUT:
452 print_warning(f"Failed to join {self} in {format_duration(dt)}")
453 break
454
455
456 def get_running(workers: list[WorkerThread]) -> str | None:
457 running: list[str] = []
458 for worker in workers:
459 test_name = worker.test_name
460 if test_name == _NOT_RUNNING:
461 continue
462 dt = time.monotonic() - worker.start_time
463 if dt >= PROGRESS_MIN_TIME:
464 text = f'{test_name} ({format_duration(dt)})'
465 running.append(text)
466 if not running:
467 return None
468 return f"running ({len(running)}): {', '.join(running)}"
469
470
471 class RunWorkers:
472 def __init__(self, num_workers: int, runtests: RunTests,
473 logger: Logger, results: TestResults) -> None:
474 self.num_workers = num_workers
475 self.runtests = runtests
476 self.log = logger.log
477 self.display_progress = logger.display_progress
478 self.results: TestResults = results
479 self.live_worker_count = 0
480
481 self.output: queue.Queue[QueueContent] = queue.Queue()
482 tests_iter = runtests.iter_tests()
483 self.pending = MultiprocessIterator(tests_iter)
484 self.timeout = runtests.timeout
485 if self.timeout is not None:
486 # Rely on faulthandler to kill a worker process. This timouet is
487 # when faulthandler fails to kill a worker process. Give a maximum
488 # of 5 minutes to faulthandler to kill the worker.
489 self.worker_timeout: float | None = min(self.timeout * 1.5, self.timeout + 5 * 60)
490 else:
491 self.worker_timeout = None
492 self.workers: list[WorkerThread] = []
493
494 jobs = self.runtests.get_jobs()
495 if jobs is not None:
496 # Don't spawn more threads than the number of jobs:
497 # these worker threads would never get anything to do.
498 self.num_workers = min(self.num_workers, jobs)
499
500 def start_workers(self) -> None:
501 self.workers = [WorkerThread(index, self)
502 for index in range(1, self.num_workers + 1)]
503 jobs = self.runtests.get_jobs()
504 if jobs is not None:
505 tests = count(jobs, 'test')
506 else:
507 tests = 'tests'
508 nworkers = len(self.workers)
509 processes = plural(nworkers, "process", "processes")
510 msg = (f"Run {tests} in parallel using "
511 f"{nworkers} worker {processes}")
512 if self.timeout and self.worker_timeout is not None:
513 msg += (" (timeout: %s, worker timeout: %s)"
514 % (format_duration(self.timeout),
515 format_duration(self.worker_timeout)))
516 self.log(msg)
517 for worker in self.workers:
518 worker.start()
519 self.live_worker_count += 1
520
521 def stop_workers(self) -> None:
522 start_time = time.monotonic()
523 for worker in self.workers:
524 worker.stop()
525 for worker in self.workers:
526 worker.wait_stopped(start_time)
527
528 def _get_result(self) -> QueueOutput | None:
529 pgo = self.runtests.pgo
530 use_faulthandler = (self.timeout is not None)
531
532 # bpo-46205: check the status of workers every iteration to avoid
533 # waiting forever on an empty queue.
534 while self.live_worker_count > 0:
535 if use_faulthandler:
536 faulthandler.dump_traceback_later(MAIN_PROCESS_TIMEOUT,
537 exit=True)
538
539 # wait for a thread
540 try:
541 result = self.output.get(timeout=PROGRESS_UPDATE)
542 if isinstance(result, WorkerThreadExited):
543 self.live_worker_count -= 1
544 continue
545 return result
546 except queue.Empty:
547 pass
548
549 if not pgo:
550 # display progress
551 running = get_running(self.workers)
552 if running:
553 self.log(running)
554 return None
555
556 def display_result(self, mp_result: MultiprocessResult) -> None:
557 result = mp_result.result
558 pgo = self.runtests.pgo
559
560 text = str(result)
561 if mp_result.err_msg:
562 # WORKER_BUG
563 text += ' (%s)' % mp_result.err_msg
564 elif (result.duration and result.duration >= PROGRESS_MIN_TIME and not pgo):
565 text += ' (%s)' % format_duration(result.duration)
566 if not pgo:
567 running = get_running(self.workers)
568 if running:
569 text += f' -- {running}'
570 self.display_progress(self.test_index, text)
571
572 def _process_result(self, item: QueueOutput) -> TestResult:
573 """Returns True if test runner must stop."""
574 if item[0]:
575 # Thread got an exception
576 format_exc = item[1]
577 print_warning(f"regrtest worker thread failed: {format_exc}")
578 result = TestResult("<regrtest worker>", state=State.WORKER_BUG)
579 self.results.accumulate_result(result, self.runtests)
580 return result
581
582 self.test_index += 1
583 mp_result = item[1]
584 result = mp_result.result
585 self.results.accumulate_result(result, self.runtests)
586 self.display_result(mp_result)
587
588 # Display worker stdout
589 if not self.runtests.output_on_failure:
590 show_stdout = True
591 else:
592 # --verbose3 ignores stdout on success
593 show_stdout = (result.state != State.PASSED)
594 if show_stdout:
595 stdout = mp_result.worker_stdout
596 if stdout:
597 print(stdout, flush=True)
598
599 return result
600
601 def run(self) -> None:
602 fail_fast = self.runtests.fail_fast
603 fail_env_changed = self.runtests.fail_env_changed
604
605 self.start_workers()
606
607 self.test_index = 0
608 try:
609 while True:
610 item = self._get_result()
611 if item is None:
612 break
613
614 result = self._process_result(item)
615 if result.must_stop(fail_fast, fail_env_changed):
616 break
617 except KeyboardInterrupt:
618 print()
619 self.results.interrupted = True
620 finally:
621 if self.timeout is not None:
622 faulthandler.cancel_dump_traceback_later()
623
624 # Always ensure that all worker processes are no longer
625 # worker when we exit this function
626 self.pending.stop()
627 self.stop_workers()