| 1 |
"""Selector event loop for Unix with signal handling.""" |
| 2 |
|
| 3 |
import errno |
| 4 |
import io |
| 5 |
import itertools |
| 6 |
import os |
| 7 |
import selectors |
| 8 |
import signal |
| 9 |
import socket |
| 10 |
import stat |
| 11 |
import subprocess |
| 12 |
import sys |
| 13 |
import threading |
| 14 |
import warnings |
| 15 |
|
| 16 |
from . import base_events |
| 17 |
from . import base_subprocess |
| 18 |
from . import constants |
| 19 |
from . import coroutines |
| 20 |
from . import events |
| 21 |
from . import exceptions |
| 22 |
from . import futures |
| 23 |
from . import selector_events |
| 24 |
from . import tasks |
| 25 |
from . import transports |
| 26 |
from .log import logger |
| 27 |
|
| 28 |
|
| 29 |
__all__ = ( |
| 30 |
'SelectorEventLoop', |
| 31 |
'AbstractChildWatcher', 'SafeChildWatcher', |
| 32 |
'FastChildWatcher', 'PidfdChildWatcher', |
| 33 |
'MultiLoopChildWatcher', 'ThreadedChildWatcher', |
| 34 |
'DefaultEventLoopPolicy', |
| 35 |
'EventLoop', |
| 36 |
) |
| 37 |
|
| 38 |
|
| 39 |
if sys.platform == 'win32': # pragma: no cover |
| 40 |
raise ImportError('Signals are not really supported on Windows') |
| 41 |
|
| 42 |
|
| 43 |
def _sighandler_noop(signum, frame): |
| 44 |
"""Dummy signal handler.""" |
| 45 |
pass |
| 46 |
|
| 47 |
|
| 48 |
def waitstatus_to_exitcode(status): |
| 49 |
try: |
| 50 |
return os.waitstatus_to_exitcode(status) |
| 51 |
except ValueError: |
| 52 |
# The child exited, but we don't understand its status. |
| 53 |
# This shouldn't happen, but if it does, let's just |
| 54 |
# return that status; perhaps that helps debug it. |
| 55 |
return status |
| 56 |
|
| 57 |
|
| 58 |
class _UnixSelectorEventLoop(selector_events.BaseSelectorEventLoop): |
| 59 |
"""Unix event loop. |
| 60 |
|
| 61 |
Adds signal handling and UNIX Domain Socket support to |
| 62 |
SelectorEventLoop. |
| 63 |
""" |
| 64 |
|
| 65 |
def __init__(self, selector=None): |
| 66 |
super().__init__(selector) |
| 67 |
self._signal_handlers = {} |
| 68 |
self._unix_server_sockets = {} |
| 69 |
|
| 70 |
def close(self): |
| 71 |
super().close() |
| 72 |
if not sys.is_finalizing(): |
| 73 |
for sig in list(self._signal_handlers): |
| 74 |
self.remove_signal_handler(sig) |
| 75 |
else: |
| 76 |
if self._signal_handlers: |
| 77 |
warnings.warn(f"Closing the loop {self!r} " |
| 78 |
f"on interpreter shutdown " |
| 79 |
f"stage, skipping signal handlers removal", |
| 80 |
ResourceWarning, |
| 81 |
source=self) |
| 82 |
self._signal_handlers.clear() |
| 83 |
|
| 84 |
def _process_self_data(self, data): |
| 85 |
for signum in data: |
| 86 |
if not signum: |
| 87 |
# ignore null bytes written by _write_to_self() |
| 88 |
continue |
| 89 |
self._handle_signal(signum) |
| 90 |
|
| 91 |
def add_signal_handler(self, sig, callback, *args): |
| 92 |
"""Add a handler for a signal. UNIX only. |
| 93 |
|
| 94 |
Raise ValueError if the signal number is invalid or uncatchable. |
| 95 |
Raise RuntimeError if there is a problem setting up the handler. |
| 96 |
""" |
| 97 |
if (coroutines.iscoroutine(callback) or |
| 98 |
coroutines.iscoroutinefunction(callback)): |
| 99 |
raise TypeError("coroutines cannot be used " |
| 100 |
"with add_signal_handler()") |
| 101 |
self._check_signal(sig) |
| 102 |
self._check_closed() |
| 103 |
try: |
| 104 |
# set_wakeup_fd() raises ValueError if this is not the |
| 105 |
# main thread. By calling it early we ensure that an |
| 106 |
# event loop running in another thread cannot add a signal |
| 107 |
# handler. |
| 108 |
signal.set_wakeup_fd(self._csock.fileno()) |
| 109 |
except (ValueError, OSError) as exc: |
| 110 |
raise RuntimeError(str(exc)) |
| 111 |
|
| 112 |
handle = events.Handle(callback, args, self, None) |
| 113 |
self._signal_handlers[sig] = handle |
| 114 |
|
| 115 |
try: |
| 116 |
# Register a dummy signal handler to ask Python to write the signal |
| 117 |
# number in the wakeup file descriptor. _process_self_data() will |
| 118 |
# read signal numbers from this file descriptor to handle signals. |
| 119 |
signal.signal(sig, _sighandler_noop) |
| 120 |
|
| 121 |
# Set SA_RESTART to limit EINTR occurrences. |
| 122 |
signal.siginterrupt(sig, False) |
| 123 |
except OSError as exc: |
| 124 |
del self._signal_handlers[sig] |
| 125 |
if not self._signal_handlers: |
| 126 |
try: |
| 127 |
signal.set_wakeup_fd(-1) |
| 128 |
except (ValueError, OSError) as nexc: |
| 129 |
logger.info('set_wakeup_fd(-1) failed: %s', nexc) |
| 130 |
|
| 131 |
if exc.errno == errno.EINVAL: |
| 132 |
raise RuntimeError(f'sig {sig} cannot be caught') |
| 133 |
else: |
| 134 |
raise |
| 135 |
|
| 136 |
def _handle_signal(self, sig): |
| 137 |
"""Internal helper that is the actual signal handler.""" |
| 138 |
handle = self._signal_handlers.get(sig) |
| 139 |
if handle is None: |
| 140 |
return # Assume it's some race condition. |
| 141 |
if handle._cancelled: |
| 142 |
self.remove_signal_handler(sig) # Remove it properly. |
| 143 |
else: |
| 144 |
self._add_callback_signalsafe(handle) |
| 145 |
|
| 146 |
def remove_signal_handler(self, sig): |
| 147 |
"""Remove a handler for a signal. UNIX only. |
| 148 |
|
| 149 |
Return True if a signal handler was removed, False if not. |
| 150 |
""" |
| 151 |
self._check_signal(sig) |
| 152 |
try: |
| 153 |
del self._signal_handlers[sig] |
| 154 |
except KeyError: |
| 155 |
return False |
| 156 |
|
| 157 |
if sig == signal.SIGINT: |
| 158 |
handler = signal.default_int_handler |
| 159 |
else: |
| 160 |
handler = signal.SIG_DFL |
| 161 |
|
| 162 |
try: |
| 163 |
signal.signal(sig, handler) |
| 164 |
except OSError as exc: |
| 165 |
if exc.errno == errno.EINVAL: |
| 166 |
raise RuntimeError(f'sig {sig} cannot be caught') |
| 167 |
else: |
| 168 |
raise |
| 169 |
|
| 170 |
if not self._signal_handlers: |
| 171 |
try: |
| 172 |
signal.set_wakeup_fd(-1) |
| 173 |
except (ValueError, OSError) as exc: |
| 174 |
logger.info('set_wakeup_fd(-1) failed: %s', exc) |
| 175 |
|
| 176 |
return True |
| 177 |
|
| 178 |
def _check_signal(self, sig): |
| 179 |
"""Internal helper to validate a signal. |
| 180 |
|
| 181 |
Raise ValueError if the signal number is invalid or uncatchable. |
| 182 |
Raise RuntimeError if there is a problem setting up the handler. |
| 183 |
""" |
| 184 |
if not isinstance(sig, int): |
| 185 |
raise TypeError(f'sig must be an int, not {sig!r}') |
| 186 |
|
| 187 |
if sig not in signal.valid_signals(): |
| 188 |
raise ValueError(f'invalid signal number {sig}') |
| 189 |
|
| 190 |
def _make_read_pipe_transport(self, pipe, protocol, waiter=None, |
| 191 |
extra=None): |
| 192 |
return _UnixReadPipeTransport(self, pipe, protocol, waiter, extra) |
| 193 |
|
| 194 |
def _make_write_pipe_transport(self, pipe, protocol, waiter=None, |
| 195 |
extra=None): |
| 196 |
return _UnixWritePipeTransport(self, pipe, protocol, waiter, extra) |
| 197 |
|
| 198 |
async def _make_subprocess_transport(self, protocol, args, shell, |
| 199 |
stdin, stdout, stderr, bufsize, |
| 200 |
extra=None, **kwargs): |
| 201 |
with warnings.catch_warnings(): |
| 202 |
warnings.simplefilter('ignore', DeprecationWarning) |
| 203 |
watcher = events.get_child_watcher() |
| 204 |
|
| 205 |
with watcher: |
| 206 |
if not watcher.is_active(): |
| 207 |
# Check early. |
| 208 |
# Raising exception before process creation |
| 209 |
# prevents subprocess execution if the watcher |
| 210 |
# is not ready to handle it. |
| 211 |
raise RuntimeError("asyncio.get_child_watcher() is not activated, " |
| 212 |
"subprocess support is not installed.") |
| 213 |
waiter = self.create_future() |
| 214 |
transp = _UnixSubprocessTransport(self, protocol, args, shell, |
| 215 |
stdin, stdout, stderr, bufsize, |
| 216 |
waiter=waiter, extra=extra, |
| 217 |
**kwargs) |
| 218 |
watcher.add_child_handler(transp.get_pid(), |
| 219 |
self._child_watcher_callback, transp) |
| 220 |
try: |
| 221 |
await waiter |
| 222 |
except (SystemExit, KeyboardInterrupt): |
| 223 |
raise |
| 224 |
except BaseException: |
| 225 |
transp.close() |
| 226 |
await transp._wait() |
| 227 |
raise |
| 228 |
|
| 229 |
return transp |
| 230 |
|
| 231 |
def _child_watcher_callback(self, pid, returncode, transp): |
| 232 |
self.call_soon_threadsafe(transp._process_exited, returncode) |
| 233 |
|
| 234 |
async def create_unix_connection( |
| 235 |
self, protocol_factory, path=None, *, |
| 236 |
ssl=None, sock=None, |
| 237 |
server_hostname=None, |
| 238 |
ssl_handshake_timeout=None, |
| 239 |
ssl_shutdown_timeout=None): |
| 240 |
assert server_hostname is None or isinstance(server_hostname, str) |
| 241 |
if ssl: |
| 242 |
if server_hostname is None: |
| 243 |
raise ValueError( |
| 244 |
'you have to pass server_hostname when using ssl') |
| 245 |
else: |
| 246 |
if server_hostname is not None: |
| 247 |
raise ValueError('server_hostname is only meaningful with ssl') |
| 248 |
if ssl_handshake_timeout is not None: |
| 249 |
raise ValueError( |
| 250 |
'ssl_handshake_timeout is only meaningful with ssl') |
| 251 |
if ssl_shutdown_timeout is not None: |
| 252 |
raise ValueError( |
| 253 |
'ssl_shutdown_timeout is only meaningful with ssl') |
| 254 |
|
| 255 |
if path is not None: |
| 256 |
if sock is not None: |
| 257 |
raise ValueError( |
| 258 |
'path and sock can not be specified at the same time') |
| 259 |
|
| 260 |
path = os.fspath(path) |
| 261 |
sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM, 0) |
| 262 |
try: |
| 263 |
sock.setblocking(False) |
| 264 |
await self.sock_connect(sock, path) |
| 265 |
except: |
| 266 |
sock.close() |
| 267 |
raise |
| 268 |
|
| 269 |
else: |
| 270 |
if sock is None: |
| 271 |
raise ValueError('no path and sock were specified') |
| 272 |
if (sock.family != socket.AF_UNIX or |
| 273 |
sock.type != socket.SOCK_STREAM): |
| 274 |
raise ValueError( |
| 275 |
f'A UNIX Domain Stream Socket was expected, got {sock!r}') |
| 276 |
sock.setblocking(False) |
| 277 |
|
| 278 |
transport, protocol = await self._create_connection_transport( |
| 279 |
sock, protocol_factory, ssl, server_hostname, |
| 280 |
ssl_handshake_timeout=ssl_handshake_timeout, |
| 281 |
ssl_shutdown_timeout=ssl_shutdown_timeout) |
| 282 |
return transport, protocol |
| 283 |
|
| 284 |
async def create_unix_server( |
| 285 |
self, protocol_factory, path=None, *, |
| 286 |
sock=None, backlog=100, ssl=None, |
| 287 |
ssl_handshake_timeout=None, |
| 288 |
ssl_shutdown_timeout=None, |
| 289 |
start_serving=True, cleanup_socket=True): |
| 290 |
if isinstance(ssl, bool): |
| 291 |
raise TypeError('ssl argument must be an SSLContext or None') |
| 292 |
|
| 293 |
if ssl_handshake_timeout is not None and not ssl: |
| 294 |
raise ValueError( |
| 295 |
'ssl_handshake_timeout is only meaningful with ssl') |
| 296 |
|
| 297 |
if ssl_shutdown_timeout is not None and not ssl: |
| 298 |
raise ValueError( |
| 299 |
'ssl_shutdown_timeout is only meaningful with ssl') |
| 300 |
|
| 301 |
if path is not None: |
| 302 |
if sock is not None: |
| 303 |
raise ValueError( |
| 304 |
'path and sock can not be specified at the same time') |
| 305 |
|
| 306 |
path = os.fspath(path) |
| 307 |
sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) |
| 308 |
|
| 309 |
# Check for abstract socket. `str` and `bytes` paths are supported. |
| 310 |
if path[0] not in (0, '\x00'): |
| 311 |
try: |
| 312 |
if stat.S_ISSOCK(os.stat(path).st_mode): |
| 313 |
os.remove(path) |
| 314 |
except FileNotFoundError: |
| 315 |
pass |
| 316 |
except OSError as err: |
| 317 |
# Directory may have permissions only to create socket. |
| 318 |
logger.error('Unable to check or remove stale UNIX socket ' |
| 319 |
'%r: %r', path, err) |
| 320 |
|
| 321 |
try: |
| 322 |
sock.bind(path) |
| 323 |
except OSError as exc: |
| 324 |
sock.close() |
| 325 |
if exc.errno == errno.EADDRINUSE: |
| 326 |
# Let's improve the error message by adding |
| 327 |
# with what exact address it occurs. |
| 328 |
msg = f'Address {path!r} is already in use' |
| 329 |
raise OSError(errno.EADDRINUSE, msg) from None |
| 330 |
else: |
| 331 |
raise |
| 332 |
except: |
| 333 |
sock.close() |
| 334 |
raise |
| 335 |
else: |
| 336 |
if sock is None: |
| 337 |
raise ValueError( |
| 338 |
'path was not specified, and no sock specified') |
| 339 |
|
| 340 |
if (sock.family != socket.AF_UNIX or |
| 341 |
sock.type != socket.SOCK_STREAM): |
| 342 |
raise ValueError( |
| 343 |
f'A UNIX Domain Stream Socket was expected, got {sock!r}') |
| 344 |
|
| 345 |
if cleanup_socket: |
| 346 |
path = sock.getsockname() |
| 347 |
# Check for abstract socket. `str` and `bytes` paths are supported. |
| 348 |
if path[0] not in (0, '\x00'): |
| 349 |
try: |
| 350 |
self._unix_server_sockets[sock] = os.stat(path).st_ino |
| 351 |
except FileNotFoundError: |
| 352 |
pass |
| 353 |
|
| 354 |
sock.setblocking(False) |
| 355 |
server = base_events.Server(self, [sock], protocol_factory, |
| 356 |
ssl, backlog, ssl_handshake_timeout, |
| 357 |
ssl_shutdown_timeout) |
| 358 |
if start_serving: |
| 359 |
server._start_serving() |
| 360 |
# Skip one loop iteration so that all 'loop.add_reader' |
| 361 |
# go through. |
| 362 |
await tasks.sleep(0) |
| 363 |
|
| 364 |
return server |
| 365 |
|
| 366 |
async def _sock_sendfile_native(self, sock, file, offset, count): |
| 367 |
try: |
| 368 |
os.sendfile |
| 369 |
except AttributeError: |
| 370 |
raise exceptions.SendfileNotAvailableError( |
| 371 |
"os.sendfile() is not available") |
| 372 |
try: |
| 373 |
fileno = file.fileno() |
| 374 |
except (AttributeError, io.UnsupportedOperation) as err: |
| 375 |
raise exceptions.SendfileNotAvailableError("not a regular file") |
| 376 |
try: |
| 377 |
fsize = os.fstat(fileno).st_size |
| 378 |
except OSError: |
| 379 |
raise exceptions.SendfileNotAvailableError("not a regular file") |
| 380 |
blocksize = count if count else fsize |
| 381 |
if not blocksize: |
| 382 |
return 0 # empty file |
| 383 |
|
| 384 |
fut = self.create_future() |
| 385 |
self._sock_sendfile_native_impl(fut, None, sock, fileno, |
| 386 |
offset, count, blocksize, 0) |
| 387 |
return await fut |
| 388 |
|
| 389 |
def _sock_sendfile_native_impl(self, fut, registered_fd, sock, fileno, |
| 390 |
offset, count, blocksize, total_sent): |
| 391 |
fd = sock.fileno() |
| 392 |
if registered_fd is not None: |
| 393 |
# Remove the callback early. It should be rare that the |
| 394 |
# selector says the fd is ready but the call still returns |
| 395 |
# EAGAIN, and I am willing to take a hit in that case in |
| 396 |
# order to simplify the common case. |
| 397 |
self.remove_writer(registered_fd) |
| 398 |
if fut.cancelled(): |
| 399 |
self._sock_sendfile_update_filepos(fileno, offset) |
| 400 |
return |
| 401 |
if count: |
| 402 |
blocksize = count - total_sent |
| 403 |
if blocksize <= 0: |
| 404 |
self._sock_sendfile_update_filepos(fileno, offset) |
| 405 |
fut.set_result(total_sent) |
| 406 |
return |
| 407 |
|
| 408 |
# On 32-bit architectures truncate to 1GiB to avoid OverflowError |
| 409 |
blocksize = min(blocksize, sys.maxsize//2 + 1) |
| 410 |
|
| 411 |
try: |
| 412 |
sent = os.sendfile(fd, fileno, offset, blocksize) |
| 413 |
except (BlockingIOError, InterruptedError): |
| 414 |
if registered_fd is None: |
| 415 |
self._sock_add_cancellation_callback(fut, sock) |
| 416 |
self.add_writer(fd, self._sock_sendfile_native_impl, fut, |
| 417 |
fd, sock, fileno, |
| 418 |
offset, count, blocksize, total_sent) |
| 419 |
except OSError as exc: |
| 420 |
if (registered_fd is not None and |
| 421 |
exc.errno == errno.ENOTCONN and |
| 422 |
type(exc) is not ConnectionError): |
| 423 |
# If we have an ENOTCONN and this isn't a first call to |
| 424 |
# sendfile(), i.e. the connection was closed in the middle |
| 425 |
# of the operation, normalize the error to ConnectionError |
| 426 |
# to make it consistent across all Posix systems. |
| 427 |
new_exc = ConnectionError( |
| 428 |
"socket is not connected", errno.ENOTCONN) |
| 429 |
new_exc.__cause__ = exc |
| 430 |
exc = new_exc |
| 431 |
if total_sent == 0: |
| 432 |
# We can get here for different reasons, the main |
| 433 |
# one being 'file' is not a regular mmap(2)-like |
| 434 |
# file, in which case we'll fall back on using |
| 435 |
# plain send(). |
| 436 |
err = exceptions.SendfileNotAvailableError( |
| 437 |
"os.sendfile call failed") |
| 438 |
self._sock_sendfile_update_filepos(fileno, offset) |
| 439 |
fut.set_exception(err) |
| 440 |
else: |
| 441 |
self._sock_sendfile_update_filepos(fileno, offset) |
| 442 |
fut.set_exception(exc) |
| 443 |
except (SystemExit, KeyboardInterrupt): |
| 444 |
raise |
| 445 |
except BaseException as exc: |
| 446 |
self._sock_sendfile_update_filepos(fileno, offset) |
| 447 |
fut.set_exception(exc) |
| 448 |
else: |
| 449 |
if sent == 0: |
| 450 |
# EOF |
| 451 |
self._sock_sendfile_update_filepos(fileno, offset) |
| 452 |
fut.set_result(total_sent) |
| 453 |
else: |
| 454 |
offset += sent |
| 455 |
total_sent += sent |
| 456 |
if registered_fd is None: |
| 457 |
self._sock_add_cancellation_callback(fut, sock) |
| 458 |
self.add_writer(fd, self._sock_sendfile_native_impl, fut, |
| 459 |
fd, sock, fileno, |
| 460 |
offset, count, blocksize, total_sent) |
| 461 |
|
| 462 |
def _sock_sendfile_update_filepos(self, fileno, offset): |
| 463 |
# After this helper runs, the source fd's lseek pointer is at offset." |
| 464 |
os.lseek(fileno, offset, os.SEEK_SET) |
| 465 |
|
| 466 |
def _sock_add_cancellation_callback(self, fut, sock): |
| 467 |
def cb(fut): |
| 468 |
if fut.cancelled(): |
| 469 |
fd = sock.fileno() |
| 470 |
if fd != -1: |
| 471 |
self.remove_writer(fd) |
| 472 |
fut.add_done_callback(cb) |
| 473 |
|
| 474 |
def _stop_serving(self, sock): |
| 475 |
# Is this a unix socket that needs cleanup? |
| 476 |
if sock in self._unix_server_sockets: |
| 477 |
path = sock.getsockname() |
| 478 |
else: |
| 479 |
path = None |
| 480 |
|
| 481 |
super()._stop_serving(sock) |
| 482 |
|
| 483 |
if path is not None: |
| 484 |
prev_ino = self._unix_server_sockets[sock] |
| 485 |
del self._unix_server_sockets[sock] |
| 486 |
try: |
| 487 |
if os.stat(path).st_ino == prev_ino: |
| 488 |
os.unlink(path) |
| 489 |
except FileNotFoundError: |
| 490 |
pass |
| 491 |
except OSError as err: |
| 492 |
logger.error('Unable to clean up listening UNIX socket ' |
| 493 |
'%r: %r', path, err) |
| 494 |
|
| 495 |
|
| 496 |
class _UnixReadPipeTransport(transports.ReadTransport): |
| 497 |
|
| 498 |
max_size = 256 * 1024 # max bytes we read in one event loop iteration |
| 499 |
|
| 500 |
def __init__(self, loop, pipe, protocol, waiter=None, extra=None): |
| 501 |
super().__init__(extra) |
| 502 |
self._extra['pipe'] = pipe |
| 503 |
self._loop = loop |
| 504 |
self._pipe = pipe |
| 505 |
self._fileno = pipe.fileno() |
| 506 |
self._protocol = protocol |
| 507 |
self._closing = False |
| 508 |
self._paused = False |
| 509 |
|
| 510 |
mode = os.fstat(self._fileno).st_mode |
| 511 |
if not (stat.S_ISFIFO(mode) or |
| 512 |
stat.S_ISSOCK(mode) or |
| 513 |
stat.S_ISCHR(mode)): |
| 514 |
self._pipe = None |
| 515 |
self._fileno = None |
| 516 |
self._protocol = None |
| 517 |
raise ValueError("Pipe transport is for pipes/sockets only.") |
| 518 |
|
| 519 |
os.set_blocking(self._fileno, False) |
| 520 |
|
| 521 |
self._loop.call_soon(self._protocol.connection_made, self) |
| 522 |
# only start reading when connection_made() has been called |
| 523 |
self._loop.call_soon(self._add_reader, |
| 524 |
self._fileno, self._read_ready) |
| 525 |
if waiter is not None: |
| 526 |
# only wake up the waiter when connection_made() has been called |
| 527 |
self._loop.call_soon(futures._set_result_unless_cancelled, |
| 528 |
waiter, None) |
| 529 |
|
| 530 |
def _add_reader(self, fd, callback): |
| 531 |
if not self.is_reading(): |
| 532 |
return |
| 533 |
self._loop._add_reader(fd, callback) |
| 534 |
|
| 535 |
def is_reading(self): |
| 536 |
return not self._paused and not self._closing |
| 537 |
|
| 538 |
def __repr__(self): |
| 539 |
info = [self.__class__.__name__] |
| 540 |
if self._pipe is None: |
| 541 |
info.append('closed') |
| 542 |
elif self._closing: |
| 543 |
info.append('closing') |
| 544 |
info.append(f'fd={self._fileno}') |
| 545 |
selector = getattr(self._loop, '_selector', None) |
| 546 |
if self._pipe is not None and selector is not None: |
| 547 |
polling = selector_events._test_selector_event( |
| 548 |
selector, self._fileno, selectors.EVENT_READ) |
| 549 |
if polling: |
| 550 |
info.append('polling') |
| 551 |
else: |
| 552 |
info.append('idle') |
| 553 |
elif self._pipe is not None: |
| 554 |
info.append('open') |
| 555 |
else: |
| 556 |
info.append('closed') |
| 557 |
return '<{}>'.format(' '.join(info)) |
| 558 |
|
| 559 |
def _read_ready(self): |
| 560 |
try: |
| 561 |
data = os.read(self._fileno, self.max_size) |
| 562 |
except (BlockingIOError, InterruptedError): |
| 563 |
pass |
| 564 |
except OSError as exc: |
| 565 |
self._fatal_error(exc, 'Fatal read error on pipe transport') |
| 566 |
else: |
| 567 |
if data: |
| 568 |
self._protocol.data_received(data) |
| 569 |
else: |
| 570 |
if self._loop.get_debug(): |
| 571 |
logger.info("%r was closed by peer", self) |
| 572 |
self._closing = True |
| 573 |
self._loop._remove_reader(self._fileno) |
| 574 |
self._loop.call_soon(self._protocol.eof_received) |
| 575 |
self._loop.call_soon(self._call_connection_lost, None) |
| 576 |
|
| 577 |
def pause_reading(self): |
| 578 |
if not self.is_reading(): |
| 579 |
return |
| 580 |
self._paused = True |
| 581 |
self._loop._remove_reader(self._fileno) |
| 582 |
if self._loop.get_debug(): |
| 583 |
logger.debug("%r pauses reading", self) |
| 584 |
|
| 585 |
def resume_reading(self): |
| 586 |
if self._closing or not self._paused: |
| 587 |
return |
| 588 |
self._paused = False |
| 589 |
self._loop._add_reader(self._fileno, self._read_ready) |
| 590 |
if self._loop.get_debug(): |
| 591 |
logger.debug("%r resumes reading", self) |
| 592 |
|
| 593 |
def set_protocol(self, protocol): |
| 594 |
self._protocol = protocol |
| 595 |
|
| 596 |
def get_protocol(self): |
| 597 |
return self._protocol |
| 598 |
|
| 599 |
def is_closing(self): |
| 600 |
return self._closing |
| 601 |
|
| 602 |
def close(self): |
| 603 |
if not self._closing: |
| 604 |
self._close(None) |
| 605 |
|
| 606 |
def __del__(self, _warn=warnings.warn): |
| 607 |
if self._pipe is not None: |
| 608 |
_warn(f"unclosed transport {self!r}", ResourceWarning, source=self) |
| 609 |
self._pipe.close() |
| 610 |
|
| 611 |
def _fatal_error(self, exc, message='Fatal error on pipe transport'): |
| 612 |
# should be called by exception handler only |
| 613 |
if (isinstance(exc, OSError) and exc.errno == errno.EIO): |
| 614 |
if self._loop.get_debug(): |
| 615 |
logger.debug("%r: %s", self, message, exc_info=True) |
| 616 |
else: |
| 617 |
self._loop.call_exception_handler({ |
| 618 |
'message': message, |
| 619 |
'exception': exc, |
| 620 |
'transport': self, |
| 621 |
'protocol': self._protocol, |
| 622 |
}) |
| 623 |
self._close(exc) |
| 624 |
|
| 625 |
def _close(self, exc): |
| 626 |
self._closing = True |
| 627 |
self._loop._remove_reader(self._fileno) |
| 628 |
self._loop.call_soon(self._call_connection_lost, exc) |
| 629 |
|
| 630 |
def _call_connection_lost(self, exc): |
| 631 |
try: |
| 632 |
self._protocol.connection_lost(exc) |
| 633 |
finally: |
| 634 |
self._pipe.close() |
| 635 |
self._pipe = None |
| 636 |
self._protocol = None |
| 637 |
self._loop = None |
| 638 |
|
| 639 |
|
| 640 |
class _UnixWritePipeTransport(transports._FlowControlMixin, |
| 641 |
transports.WriteTransport): |
| 642 |
|
| 643 |
def __init__(self, loop, pipe, protocol, waiter=None, extra=None): |
| 644 |
super().__init__(extra, loop) |
| 645 |
self._extra['pipe'] = pipe |
| 646 |
self._pipe = pipe |
| 647 |
self._fileno = pipe.fileno() |
| 648 |
self._protocol = protocol |
| 649 |
self._buffer = bytearray() |
| 650 |
self._conn_lost = 0 |
| 651 |
self._closing = False # Set when close() or write_eof() called. |
| 652 |
|
| 653 |
pipe_stat = os.fstat(self._fileno) |
| 654 |
mode = pipe_stat.st_mode |
| 655 |
is_char = stat.S_ISCHR(mode) |
| 656 |
is_fifo = stat.S_ISFIFO(mode) |
| 657 |
is_socket = stat.S_ISSOCK(mode) |
| 658 |
if not (is_char or is_fifo or is_socket): |
| 659 |
self._pipe = None |
| 660 |
self._fileno = None |
| 661 |
self._protocol = None |
| 662 |
raise ValueError("Pipe transport is only for " |
| 663 |
"pipes, sockets and character devices") |
| 664 |
|
| 665 |
os.set_blocking(self._fileno, False) |
| 666 |
self._loop.call_soon(self._protocol.connection_made, self) |
| 667 |
|
| 668 |
# On AIX, the reader trick (to be notified when the read end of the |
| 669 |
# socket is closed) only works for sockets. On other platforms it |
| 670 |
# works for pipes and sockets. (Exception: OS X 10.4? Issue #19294.) |
| 671 |
# On macOS and Solaris, the trick misfires for named FIFOs (but not for |
| 672 |
# pipes created with os.pipe(), which have st_nlink == 0): the write |
| 673 |
# end polls as readable whenever unread data sits in the FIFO, and no |
| 674 |
# event is delivered when the read end is closed, so it can only |
| 675 |
# ever report a false disconnection (gh-145030). The same XNU |
| 676 |
# behaviour applies on iOS/tvOS/watchOS (sys.platform is not |
| 677 |
# "darwin" there). |
| 678 |
is_named_fifo_without_close_event = ( |
| 679 |
sys.platform in {"darwin", "ios", "tvos", "watchos", "sunos5"} |
| 680 |
and is_fifo and pipe_stat.st_nlink > 0) |
| 681 |
if is_socket or (is_fifo |
| 682 |
and not sys.platform.startswith("aix") |
| 683 |
and not is_named_fifo_without_close_event): |
| 684 |
# only start reading when connection_made() has been called |
| 685 |
self._loop.call_soon(self._loop._add_reader, |
| 686 |
self._fileno, self._read_ready) |
| 687 |
|
| 688 |
if waiter is not None: |
| 689 |
# only wake up the waiter when connection_made() has been called |
| 690 |
self._loop.call_soon(futures._set_result_unless_cancelled, |
| 691 |
waiter, None) |
| 692 |
|
| 693 |
def __repr__(self): |
| 694 |
info = [self.__class__.__name__] |
| 695 |
if self._pipe is None: |
| 696 |
info.append('closed') |
| 697 |
elif self._closing: |
| 698 |
info.append('closing') |
| 699 |
info.append(f'fd={self._fileno}') |
| 700 |
selector = getattr(self._loop, '_selector', None) |
| 701 |
if self._pipe is not None and selector is not None: |
| 702 |
polling = selector_events._test_selector_event( |
| 703 |
selector, self._fileno, selectors.EVENT_WRITE) |
| 704 |
if polling: |
| 705 |
info.append('polling') |
| 706 |
else: |
| 707 |
info.append('idle') |
| 708 |
|
| 709 |
bufsize = self.get_write_buffer_size() |
| 710 |
info.append(f'bufsize={bufsize}') |
| 711 |
elif self._pipe is not None: |
| 712 |
info.append('open') |
| 713 |
else: |
| 714 |
info.append('closed') |
| 715 |
return '<{}>'.format(' '.join(info)) |
| 716 |
|
| 717 |
def get_write_buffer_size(self): |
| 718 |
return len(self._buffer) |
| 719 |
|
| 720 |
def _read_ready(self): |
| 721 |
# Pipe was closed by peer. |
| 722 |
if self._loop.get_debug(): |
| 723 |
logger.info("%r was closed by peer", self) |
| 724 |
if self._buffer: |
| 725 |
self._close(BrokenPipeError()) |
| 726 |
else: |
| 727 |
self._close() |
| 728 |
|
| 729 |
def write(self, data): |
| 730 |
assert isinstance(data, (bytes, bytearray, memoryview)), repr(data) |
| 731 |
if isinstance(data, bytearray): |
| 732 |
data = memoryview(data) |
| 733 |
if not data: |
| 734 |
return |
| 735 |
|
| 736 |
if self._conn_lost or self._closing: |
| 737 |
if self._conn_lost >= constants.LOG_THRESHOLD_FOR_CONNLOST_WRITES: |
| 738 |
logger.warning('pipe closed by peer or ' |
| 739 |
'os.write(pipe, data) raised exception.') |
| 740 |
self._conn_lost += 1 |
| 741 |
return |
| 742 |
|
| 743 |
if not self._buffer: |
| 744 |
# Attempt to send it right away first. |
| 745 |
try: |
| 746 |
n = os.write(self._fileno, data) |
| 747 |
except (BlockingIOError, InterruptedError): |
| 748 |
n = 0 |
| 749 |
except (SystemExit, KeyboardInterrupt): |
| 750 |
raise |
| 751 |
except BaseException as exc: |
| 752 |
self._conn_lost += 1 |
| 753 |
self._fatal_error(exc, 'Fatal write error on pipe transport') |
| 754 |
return |
| 755 |
if n == len(data): |
| 756 |
return |
| 757 |
elif n > 0: |
| 758 |
data = memoryview(data)[n:] |
| 759 |
self._loop._add_writer(self._fileno, self._write_ready) |
| 760 |
|
| 761 |
self._buffer += data |
| 762 |
self._maybe_pause_protocol() |
| 763 |
|
| 764 |
def _write_ready(self): |
| 765 |
assert self._buffer, 'Data should not be empty' |
| 766 |
|
| 767 |
try: |
| 768 |
n = os.write(self._fileno, self._buffer) |
| 769 |
except (BlockingIOError, InterruptedError): |
| 770 |
pass |
| 771 |
except (SystemExit, KeyboardInterrupt): |
| 772 |
raise |
| 773 |
except BaseException as exc: |
| 774 |
self._buffer.clear() |
| 775 |
self._conn_lost += 1 |
| 776 |
# Remove writer here, _fatal_error() doesn't it |
| 777 |
# because _buffer is empty. |
| 778 |
self._loop._remove_writer(self._fileno) |
| 779 |
self._fatal_error(exc, 'Fatal write error on pipe transport') |
| 780 |
else: |
| 781 |
if n == len(self._buffer): |
| 782 |
self._buffer.clear() |
| 783 |
self._loop._remove_writer(self._fileno) |
| 784 |
self._maybe_resume_protocol() # May append to buffer. |
| 785 |
if self._closing: |
| 786 |
self._loop._remove_reader(self._fileno) |
| 787 |
self._call_connection_lost(None) |
| 788 |
return |
| 789 |
elif n > 0: |
| 790 |
del self._buffer[:n] |
| 791 |
|
| 792 |
def can_write_eof(self): |
| 793 |
return True |
| 794 |
|
| 795 |
def write_eof(self): |
| 796 |
if self._closing: |
| 797 |
return |
| 798 |
assert self._pipe |
| 799 |
self._closing = True |
| 800 |
if not self._buffer: |
| 801 |
self._loop._remove_reader(self._fileno) |
| 802 |
self._loop.call_soon(self._call_connection_lost, None) |
| 803 |
|
| 804 |
def set_protocol(self, protocol): |
| 805 |
self._protocol = protocol |
| 806 |
|
| 807 |
def get_protocol(self): |
| 808 |
return self._protocol |
| 809 |
|
| 810 |
def is_closing(self): |
| 811 |
return self._closing |
| 812 |
|
| 813 |
def close(self): |
| 814 |
if self._pipe is not None and not self._closing: |
| 815 |
# write_eof is all what we needed to close the write pipe |
| 816 |
self.write_eof() |
| 817 |
|
| 818 |
def __del__(self, _warn=warnings.warn): |
| 819 |
if self._pipe is not None: |
| 820 |
_warn(f"unclosed transport {self!r}", ResourceWarning, source=self) |
| 821 |
self._pipe.close() |
| 822 |
|
| 823 |
def abort(self): |
| 824 |
self._close(None) |
| 825 |
|
| 826 |
def _fatal_error(self, exc, message='Fatal error on pipe transport'): |
| 827 |
# should be called by exception handler only |
| 828 |
if isinstance(exc, OSError): |
| 829 |
if self._loop.get_debug(): |
| 830 |
logger.debug("%r: %s", self, message, exc_info=True) |
| 831 |
else: |
| 832 |
self._loop.call_exception_handler({ |
| 833 |
'message': message, |
| 834 |
'exception': exc, |
| 835 |
'transport': self, |
| 836 |
'protocol': self._protocol, |
| 837 |
}) |
| 838 |
self._close(exc) |
| 839 |
|
| 840 |
def _close(self, exc=None): |
| 841 |
self._closing = True |
| 842 |
if self._buffer: |
| 843 |
self._loop._remove_writer(self._fileno) |
| 844 |
self._buffer.clear() |
| 845 |
self._loop._remove_reader(self._fileno) |
| 846 |
self._loop.call_soon(self._call_connection_lost, exc) |
| 847 |
|
| 848 |
def _call_connection_lost(self, exc): |
| 849 |
try: |
| 850 |
self._protocol.connection_lost(exc) |
| 851 |
finally: |
| 852 |
self._pipe.close() |
| 853 |
self._pipe = None |
| 854 |
self._protocol = None |
| 855 |
self._loop = None |
| 856 |
|
| 857 |
|
| 858 |
class _UnixSubprocessTransport(base_subprocess.BaseSubprocessTransport): |
| 859 |
|
| 860 |
def _start(self, args, shell, stdin, stdout, stderr, bufsize, **kwargs): |
| 861 |
stdin_w = None |
| 862 |
if stdin == subprocess.PIPE and sys.platform.startswith('aix'): |
| 863 |
# Use a socket pair for stdin on AIX, since it does not |
| 864 |
# support selecting read events on the write end of a |
| 865 |
# socket (which we use in order to detect closing of the |
| 866 |
# other end). |
| 867 |
stdin, stdin_w = socket.socketpair() |
| 868 |
try: |
| 869 |
self._proc = subprocess.Popen( |
| 870 |
args, shell=shell, stdin=stdin, stdout=stdout, stderr=stderr, |
| 871 |
universal_newlines=False, bufsize=bufsize, **kwargs) |
| 872 |
if stdin_w is not None: |
| 873 |
stdin.close() |
| 874 |
self._proc.stdin = open(stdin_w.detach(), 'wb', buffering=bufsize) |
| 875 |
stdin_w = None |
| 876 |
finally: |
| 877 |
if stdin_w is not None: |
| 878 |
stdin.close() |
| 879 |
stdin_w.close() |
| 880 |
|
| 881 |
|
| 882 |
class AbstractChildWatcher: |
| 883 |
"""Abstract base class for monitoring child processes. |
| 884 |
|
| 885 |
Objects derived from this class monitor a collection of subprocesses and |
| 886 |
report their termination or interruption by a signal. |
| 887 |
|
| 888 |
New callbacks are registered with .add_child_handler(). Starting a new |
| 889 |
process must be done within a 'with' block to allow the watcher to suspend |
| 890 |
its activity until the new process if fully registered (this is needed to |
| 891 |
prevent a race condition in some implementations). |
| 892 |
|
| 893 |
Example: |
| 894 |
with watcher: |
| 895 |
proc = subprocess.Popen("sleep 1") |
| 896 |
watcher.add_child_handler(proc.pid, callback) |
| 897 |
|
| 898 |
Notes: |
| 899 |
Implementations of this class must be thread-safe. |
| 900 |
|
| 901 |
Since child watcher objects may catch the SIGCHLD signal and call |
| 902 |
waitpid(-1), there should be only one active object per process. |
| 903 |
""" |
| 904 |
|
| 905 |
def __init_subclass__(cls) -> None: |
| 906 |
if cls.__module__ != __name__: |
| 907 |
warnings._deprecated("AbstractChildWatcher", |
| 908 |
"{name!r} is deprecated as of Python 3.12 and will be " |
| 909 |
"removed in Python {remove}.", |
| 910 |
remove=(3, 14)) |
| 911 |
|
| 912 |
def add_child_handler(self, pid, callback, *args): |
| 913 |
"""Register a new child handler. |
| 914 |
|
| 915 |
Arrange for callback(pid, returncode, *args) to be called when |
| 916 |
process 'pid' terminates. Specifying another callback for the same |
| 917 |
process replaces the previous handler. |
| 918 |
|
| 919 |
Note: callback() must be thread-safe. |
| 920 |
""" |
| 921 |
raise NotImplementedError() |
| 922 |
|
| 923 |
def remove_child_handler(self, pid): |
| 924 |
"""Removes the handler for process 'pid'. |
| 925 |
|
| 926 |
The function returns True if the handler was successfully removed, |
| 927 |
False if there was nothing to remove.""" |
| 928 |
|
| 929 |
raise NotImplementedError() |
| 930 |
|
| 931 |
def attach_loop(self, loop): |
| 932 |
"""Attach the watcher to an event loop. |
| 933 |
|
| 934 |
If the watcher was previously attached to an event loop, then it is |
| 935 |
first detached before attaching to the new loop. |
| 936 |
|
| 937 |
Note: loop may be None. |
| 938 |
""" |
| 939 |
raise NotImplementedError() |
| 940 |
|
| 941 |
def close(self): |
| 942 |
"""Close the watcher. |
| 943 |
|
| 944 |
This must be called to make sure that any underlying resource is freed. |
| 945 |
""" |
| 946 |
raise NotImplementedError() |
| 947 |
|
| 948 |
def is_active(self): |
| 949 |
"""Return ``True`` if the watcher is active and is used by the event loop. |
| 950 |
|
| 951 |
Return True if the watcher is installed and ready to handle process exit |
| 952 |
notifications. |
| 953 |
|
| 954 |
""" |
| 955 |
raise NotImplementedError() |
| 956 |
|
| 957 |
def __enter__(self): |
| 958 |
"""Enter the watcher's context and allow starting new processes |
| 959 |
|
| 960 |
This function must return self""" |
| 961 |
raise NotImplementedError() |
| 962 |
|
| 963 |
def __exit__(self, a, b, c): |
| 964 |
"""Exit the watcher's context""" |
| 965 |
raise NotImplementedError() |
| 966 |
|
| 967 |
|
| 968 |
class PidfdChildWatcher(AbstractChildWatcher): |
| 969 |
"""Child watcher implementation using Linux's pid file descriptors. |
| 970 |
|
| 971 |
This child watcher polls process file descriptors (pidfds) to await child |
| 972 |
process termination. In some respects, PidfdChildWatcher is a "Goldilocks" |
| 973 |
child watcher implementation. It doesn't require signals or threads, doesn't |
| 974 |
interfere with any processes launched outside the event loop, and scales |
| 975 |
linearly with the number of subprocesses launched by the event loop. The |
| 976 |
main disadvantage is that pidfds are specific to Linux, and only work on |
| 977 |
recent (5.3+) kernels. |
| 978 |
""" |
| 979 |
|
| 980 |
def __enter__(self): |
| 981 |
return self |
| 982 |
|
| 983 |
def __exit__(self, exc_type, exc_value, exc_traceback): |
| 984 |
pass |
| 985 |
|
| 986 |
def is_active(self): |
| 987 |
return True |
| 988 |
|
| 989 |
def close(self): |
| 990 |
pass |
| 991 |
|
| 992 |
def attach_loop(self, loop): |
| 993 |
pass |
| 994 |
|
| 995 |
def add_child_handler(self, pid, callback, *args): |
| 996 |
loop = events.get_running_loop() |
| 997 |
pidfd = os.pidfd_open(pid) |
| 998 |
loop._add_reader(pidfd, self._do_wait, pid, pidfd, callback, args) |
| 999 |
|
| 1000 |
def _do_wait(self, pid, pidfd, callback, args): |
| 1001 |
loop = events.get_running_loop() |
| 1002 |
loop._remove_reader(pidfd) |
| 1003 |
try: |
| 1004 |
_, status = os.waitpid(pid, 0) |
| 1005 |
except ChildProcessError: |
| 1006 |
# The child process is already reaped |
| 1007 |
# (may happen if waitpid() is called elsewhere). |
| 1008 |
returncode = 255 |
| 1009 |
logger.warning( |
| 1010 |
"child process pid %d exit status already read: " |
| 1011 |
" will report returncode 255", |
| 1012 |
pid) |
| 1013 |
else: |
| 1014 |
returncode = waitstatus_to_exitcode(status) |
| 1015 |
|
| 1016 |
os.close(pidfd) |
| 1017 |
callback(pid, returncode, *args) |
| 1018 |
|
| 1019 |
def remove_child_handler(self, pid): |
| 1020 |
# asyncio never calls remove_child_handler() !!! |
| 1021 |
# The method is no-op but is implemented because |
| 1022 |
# abstract base classes require it. |
| 1023 |
return True |
| 1024 |
|
| 1025 |
|
| 1026 |
class BaseChildWatcher(AbstractChildWatcher): |
| 1027 |
|
| 1028 |
def __init__(self): |
| 1029 |
self._loop = None |
| 1030 |
self._callbacks = {} |
| 1031 |
|
| 1032 |
def close(self): |
| 1033 |
self.attach_loop(None) |
| 1034 |
|
| 1035 |
def is_active(self): |
| 1036 |
return self._loop is not None and self._loop.is_running() |
| 1037 |
|
| 1038 |
def _do_waitpid(self, expected_pid): |
| 1039 |
raise NotImplementedError() |
| 1040 |
|
| 1041 |
def _do_waitpid_all(self): |
| 1042 |
raise NotImplementedError() |
| 1043 |
|
| 1044 |
def attach_loop(self, loop): |
| 1045 |
assert loop is None or isinstance(loop, events.AbstractEventLoop) |
| 1046 |
|
| 1047 |
if self._loop is not None and loop is None and self._callbacks: |
| 1048 |
warnings.warn( |
| 1049 |
'A loop is being detached ' |
| 1050 |
'from a child watcher with pending handlers', |
| 1051 |
RuntimeWarning) |
| 1052 |
|
| 1053 |
if self._loop is not None: |
| 1054 |
self._loop.remove_signal_handler(signal.SIGCHLD) |
| 1055 |
|
| 1056 |
self._loop = loop |
| 1057 |
if loop is not None: |
| 1058 |
loop.add_signal_handler(signal.SIGCHLD, self._sig_chld) |
| 1059 |
|
| 1060 |
# Prevent a race condition in case a child terminated |
| 1061 |
# during the switch. |
| 1062 |
self._do_waitpid_all() |
| 1063 |
|
| 1064 |
def _sig_chld(self): |
| 1065 |
try: |
| 1066 |
self._do_waitpid_all() |
| 1067 |
except (SystemExit, KeyboardInterrupt): |
| 1068 |
raise |
| 1069 |
except BaseException as exc: |
| 1070 |
# self._loop should always be available here |
| 1071 |
# as '_sig_chld' is added as a signal handler |
| 1072 |
# in 'attach_loop' |
| 1073 |
self._loop.call_exception_handler({ |
| 1074 |
'message': 'Unknown exception in SIGCHLD handler', |
| 1075 |
'exception': exc, |
| 1076 |
}) |
| 1077 |
|
| 1078 |
|
| 1079 |
class SafeChildWatcher(BaseChildWatcher): |
| 1080 |
"""'Safe' child watcher implementation. |
| 1081 |
|
| 1082 |
This implementation avoids disrupting other code spawning processes by |
| 1083 |
polling explicitly each process in the SIGCHLD handler instead of calling |
| 1084 |
os.waitpid(-1). |
| 1085 |
|
| 1086 |
This is a safe solution but it has a significant overhead when handling a |
| 1087 |
big number of children (O(n) each time SIGCHLD is raised) |
| 1088 |
""" |
| 1089 |
|
| 1090 |
def __init__(self): |
| 1091 |
super().__init__() |
| 1092 |
warnings._deprecated("SafeChildWatcher", |
| 1093 |
"{name!r} is deprecated as of Python 3.12 and will be " |
| 1094 |
"removed in Python {remove}.", |
| 1095 |
remove=(3, 14)) |
| 1096 |
|
| 1097 |
def close(self): |
| 1098 |
self._callbacks.clear() |
| 1099 |
super().close() |
| 1100 |
|
| 1101 |
def __enter__(self): |
| 1102 |
return self |
| 1103 |
|
| 1104 |
def __exit__(self, a, b, c): |
| 1105 |
pass |
| 1106 |
|
| 1107 |
def add_child_handler(self, pid, callback, *args): |
| 1108 |
self._callbacks[pid] = (callback, args) |
| 1109 |
|
| 1110 |
# Prevent a race condition in case the child is already terminated. |
| 1111 |
self._do_waitpid(pid) |
| 1112 |
|
| 1113 |
def remove_child_handler(self, pid): |
| 1114 |
try: |
| 1115 |
del self._callbacks[pid] |
| 1116 |
return True |
| 1117 |
except KeyError: |
| 1118 |
return False |
| 1119 |
|
| 1120 |
def _do_waitpid_all(self): |
| 1121 |
|
| 1122 |
for pid in list(self._callbacks): |
| 1123 |
self._do_waitpid(pid) |
| 1124 |
|
| 1125 |
def _do_waitpid(self, expected_pid): |
| 1126 |
assert expected_pid > 0 |
| 1127 |
|
| 1128 |
try: |
| 1129 |
pid, status = os.waitpid(expected_pid, os.WNOHANG) |
| 1130 |
except ChildProcessError: |
| 1131 |
# The child process is already reaped |
| 1132 |
# (may happen if waitpid() is called elsewhere). |
| 1133 |
pid = expected_pid |
| 1134 |
returncode = 255 |
| 1135 |
logger.warning( |
| 1136 |
"Unknown child process pid %d, will report returncode 255", |
| 1137 |
pid) |
| 1138 |
else: |
| 1139 |
if pid == 0: |
| 1140 |
# The child process is still alive. |
| 1141 |
return |
| 1142 |
|
| 1143 |
returncode = waitstatus_to_exitcode(status) |
| 1144 |
if self._loop.get_debug(): |
| 1145 |
logger.debug('process %s exited with returncode %s', |
| 1146 |
expected_pid, returncode) |
| 1147 |
|
| 1148 |
try: |
| 1149 |
callback, args = self._callbacks.pop(pid) |
| 1150 |
except KeyError: # pragma: no cover |
| 1151 |
# May happen if .remove_child_handler() is called |
| 1152 |
# after os.waitpid() returns. |
| 1153 |
if self._loop.get_debug(): |
| 1154 |
logger.warning("Child watcher got an unexpected pid: %r", |
| 1155 |
pid, exc_info=True) |
| 1156 |
else: |
| 1157 |
callback(pid, returncode, *args) |
| 1158 |
|
| 1159 |
|
| 1160 |
class FastChildWatcher(BaseChildWatcher): |
| 1161 |
"""'Fast' child watcher implementation. |
| 1162 |
|
| 1163 |
This implementation reaps every terminated processes by calling |
| 1164 |
os.waitpid(-1) directly, possibly breaking other code spawning processes |
| 1165 |
and waiting for their termination. |
| 1166 |
|
| 1167 |
There is no noticeable overhead when handling a big number of children |
| 1168 |
(O(1) each time a child terminates). |
| 1169 |
""" |
| 1170 |
def __init__(self): |
| 1171 |
super().__init__() |
| 1172 |
self._lock = threading.Lock() |
| 1173 |
self._zombies = {} |
| 1174 |
self._forks = 0 |
| 1175 |
warnings._deprecated("FastChildWatcher", |
| 1176 |
"{name!r} is deprecated as of Python 3.12 and will be " |
| 1177 |
"removed in Python {remove}.", |
| 1178 |
remove=(3, 14)) |
| 1179 |
|
| 1180 |
def close(self): |
| 1181 |
self._callbacks.clear() |
| 1182 |
self._zombies.clear() |
| 1183 |
super().close() |
| 1184 |
|
| 1185 |
def __enter__(self): |
| 1186 |
with self._lock: |
| 1187 |
self._forks += 1 |
| 1188 |
|
| 1189 |
return self |
| 1190 |
|
| 1191 |
def __exit__(self, a, b, c): |
| 1192 |
with self._lock: |
| 1193 |
self._forks -= 1 |
| 1194 |
|
| 1195 |
if self._forks or not self._zombies: |
| 1196 |
return |
| 1197 |
|
| 1198 |
collateral_victims = str(self._zombies) |
| 1199 |
self._zombies.clear() |
| 1200 |
|
| 1201 |
logger.warning( |
| 1202 |
"Caught subprocesses termination from unknown pids: %s", |
| 1203 |
collateral_victims) |
| 1204 |
|
| 1205 |
def add_child_handler(self, pid, callback, *args): |
| 1206 |
assert self._forks, "Must use the context manager" |
| 1207 |
|
| 1208 |
with self._lock: |
| 1209 |
try: |
| 1210 |
returncode = self._zombies.pop(pid) |
| 1211 |
except KeyError: |
| 1212 |
# The child is running. |
| 1213 |
self._callbacks[pid] = callback, args |
| 1214 |
return |
| 1215 |
|
| 1216 |
# The child is dead already. We can fire the callback. |
| 1217 |
callback(pid, returncode, *args) |
| 1218 |
|
| 1219 |
def remove_child_handler(self, pid): |
| 1220 |
try: |
| 1221 |
del self._callbacks[pid] |
| 1222 |
return True |
| 1223 |
except KeyError: |
| 1224 |
return False |
| 1225 |
|
| 1226 |
def _do_waitpid_all(self): |
| 1227 |
# Because of signal coalescing, we must keep calling waitpid() as |
| 1228 |
# long as we're able to reap a child. |
| 1229 |
while True: |
| 1230 |
try: |
| 1231 |
pid, status = os.waitpid(-1, os.WNOHANG) |
| 1232 |
except ChildProcessError: |
| 1233 |
# No more child processes exist. |
| 1234 |
return |
| 1235 |
else: |
| 1236 |
if pid == 0: |
| 1237 |
# A child process is still alive. |
| 1238 |
return |
| 1239 |
|
| 1240 |
returncode = waitstatus_to_exitcode(status) |
| 1241 |
|
| 1242 |
with self._lock: |
| 1243 |
try: |
| 1244 |
callback, args = self._callbacks.pop(pid) |
| 1245 |
except KeyError: |
| 1246 |
# unknown child |
| 1247 |
if self._forks: |
| 1248 |
# It may not be registered yet. |
| 1249 |
self._zombies[pid] = returncode |
| 1250 |
if self._loop.get_debug(): |
| 1251 |
logger.debug('unknown process %s exited ' |
| 1252 |
'with returncode %s', |
| 1253 |
pid, returncode) |
| 1254 |
continue |
| 1255 |
callback = None |
| 1256 |
else: |
| 1257 |
if self._loop.get_debug(): |
| 1258 |
logger.debug('process %s exited with returncode %s', |
| 1259 |
pid, returncode) |
| 1260 |
|
| 1261 |
if callback is None: |
| 1262 |
logger.warning( |
| 1263 |
"Caught subprocess termination from unknown pid: " |
| 1264 |
"%d -> %d", pid, returncode) |
| 1265 |
else: |
| 1266 |
callback(pid, returncode, *args) |
| 1267 |
|
| 1268 |
|
| 1269 |
class MultiLoopChildWatcher(AbstractChildWatcher): |
| 1270 |
"""A watcher that doesn't require running loop in the main thread. |
| 1271 |
|
| 1272 |
This implementation registers a SIGCHLD signal handler on |
| 1273 |
instantiation (which may conflict with other code that |
| 1274 |
install own handler for this signal). |
| 1275 |
|
| 1276 |
The solution is safe but it has a significant overhead when |
| 1277 |
handling a big number of processes (*O(n)* each time a |
| 1278 |
SIGCHLD is received). |
| 1279 |
""" |
| 1280 |
|
| 1281 |
# Implementation note: |
| 1282 |
# The class keeps compatibility with AbstractChildWatcher ABC |
| 1283 |
# To achieve this it has empty attach_loop() method |
| 1284 |
# and doesn't accept explicit loop argument |
| 1285 |
# for add_child_handler()/remove_child_handler() |
| 1286 |
# but retrieves the current loop by get_running_loop() |
| 1287 |
|
| 1288 |
def __init__(self): |
| 1289 |
self._callbacks = {} |
| 1290 |
self._saved_sighandler = None |
| 1291 |
warnings._deprecated("MultiLoopChildWatcher", |
| 1292 |
"{name!r} is deprecated as of Python 3.12 and will be " |
| 1293 |
"removed in Python {remove}.", |
| 1294 |
remove=(3, 14)) |
| 1295 |
|
| 1296 |
def is_active(self): |
| 1297 |
return self._saved_sighandler is not None |
| 1298 |
|
| 1299 |
def close(self): |
| 1300 |
self._callbacks.clear() |
| 1301 |
if self._saved_sighandler is None: |
| 1302 |
return |
| 1303 |
|
| 1304 |
handler = signal.getsignal(signal.SIGCHLD) |
| 1305 |
if handler != self._sig_chld: |
| 1306 |
logger.warning("SIGCHLD handler was changed by outside code") |
| 1307 |
else: |
| 1308 |
signal.signal(signal.SIGCHLD, self._saved_sighandler) |
| 1309 |
self._saved_sighandler = None |
| 1310 |
|
| 1311 |
def __enter__(self): |
| 1312 |
return self |
| 1313 |
|
| 1314 |
def __exit__(self, exc_type, exc_val, exc_tb): |
| 1315 |
pass |
| 1316 |
|
| 1317 |
def add_child_handler(self, pid, callback, *args): |
| 1318 |
loop = events.get_running_loop() |
| 1319 |
self._callbacks[pid] = (loop, callback, args) |
| 1320 |
|
| 1321 |
# Prevent a race condition in case the child is already terminated. |
| 1322 |
self._do_waitpid(pid) |
| 1323 |
|
| 1324 |
def remove_child_handler(self, pid): |
| 1325 |
try: |
| 1326 |
del self._callbacks[pid] |
| 1327 |
return True |
| 1328 |
except KeyError: |
| 1329 |
return False |
| 1330 |
|
| 1331 |
def attach_loop(self, loop): |
| 1332 |
# Don't save the loop but initialize itself if called first time |
| 1333 |
# The reason to do it here is that attach_loop() is called from |
| 1334 |
# unix policy only for the main thread. |
| 1335 |
# Main thread is required for subscription on SIGCHLD signal |
| 1336 |
if self._saved_sighandler is not None: |
| 1337 |
return |
| 1338 |
|
| 1339 |
self._saved_sighandler = signal.signal(signal.SIGCHLD, self._sig_chld) |
| 1340 |
if self._saved_sighandler is None: |
| 1341 |
logger.warning("Previous SIGCHLD handler was set by non-Python code, " |
| 1342 |
"restore to default handler on watcher close.") |
| 1343 |
self._saved_sighandler = signal.SIG_DFL |
| 1344 |
|
| 1345 |
# Set SA_RESTART to limit EINTR occurrences. |
| 1346 |
signal.siginterrupt(signal.SIGCHLD, False) |
| 1347 |
|
| 1348 |
def _do_waitpid_all(self): |
| 1349 |
for pid in list(self._callbacks): |
| 1350 |
self._do_waitpid(pid) |
| 1351 |
|
| 1352 |
def _do_waitpid(self, expected_pid): |
| 1353 |
assert expected_pid > 0 |
| 1354 |
|
| 1355 |
try: |
| 1356 |
pid, status = os.waitpid(expected_pid, os.WNOHANG) |
| 1357 |
except ChildProcessError: |
| 1358 |
# The child process is already reaped |
| 1359 |
# (may happen if waitpid() is called elsewhere). |
| 1360 |
pid = expected_pid |
| 1361 |
returncode = 255 |
| 1362 |
logger.warning( |
| 1363 |
"Unknown child process pid %d, will report returncode 255", |
| 1364 |
pid) |
| 1365 |
debug_log = False |
| 1366 |
else: |
| 1367 |
if pid == 0: |
| 1368 |
# The child process is still alive. |
| 1369 |
return |
| 1370 |
|
| 1371 |
returncode = waitstatus_to_exitcode(status) |
| 1372 |
debug_log = True |
| 1373 |
try: |
| 1374 |
loop, callback, args = self._callbacks.pop(pid) |
| 1375 |
except KeyError: # pragma: no cover |
| 1376 |
# May happen if .remove_child_handler() is called |
| 1377 |
# after os.waitpid() returns. |
| 1378 |
logger.warning("Child watcher got an unexpected pid: %r", |
| 1379 |
pid, exc_info=True) |
| 1380 |
else: |
| 1381 |
if loop.is_closed(): |
| 1382 |
logger.warning("Loop %r that handles pid %r is closed", loop, pid) |
| 1383 |
else: |
| 1384 |
if debug_log and loop.get_debug(): |
| 1385 |
logger.debug('process %s exited with returncode %s', |
| 1386 |
expected_pid, returncode) |
| 1387 |
loop.call_soon_threadsafe(callback, pid, returncode, *args) |
| 1388 |
|
| 1389 |
def _sig_chld(self, signum, frame): |
| 1390 |
try: |
| 1391 |
self._do_waitpid_all() |
| 1392 |
except (SystemExit, KeyboardInterrupt): |
| 1393 |
raise |
| 1394 |
except BaseException: |
| 1395 |
logger.warning('Unknown exception in SIGCHLD handler', exc_info=True) |
| 1396 |
|
| 1397 |
|
| 1398 |
class ThreadedChildWatcher(AbstractChildWatcher): |
| 1399 |
"""Threaded child watcher implementation. |
| 1400 |
|
| 1401 |
The watcher uses a thread per process |
| 1402 |
for waiting for the process finish. |
| 1403 |
|
| 1404 |
It doesn't require subscription on POSIX signal |
| 1405 |
but a thread creation is not free. |
| 1406 |
|
| 1407 |
The watcher has O(1) complexity, its performance doesn't depend |
| 1408 |
on amount of spawn processes. |
| 1409 |
""" |
| 1410 |
|
| 1411 |
def __init__(self): |
| 1412 |
self._pid_counter = itertools.count(0) |
| 1413 |
self._threads = {} |
| 1414 |
|
| 1415 |
def is_active(self): |
| 1416 |
return True |
| 1417 |
|
| 1418 |
def close(self): |
| 1419 |
pass |
| 1420 |
|
| 1421 |
def __enter__(self): |
| 1422 |
return self |
| 1423 |
|
| 1424 |
def __exit__(self, exc_type, exc_val, exc_tb): |
| 1425 |
pass |
| 1426 |
|
| 1427 |
def __del__(self, _warn=warnings.warn): |
| 1428 |
threads = [thread for thread in list(self._threads.values()) |
| 1429 |
if thread.is_alive()] |
| 1430 |
if threads: |
| 1431 |
_warn(f"{self.__class__} has registered but not finished child processes", |
| 1432 |
ResourceWarning, |
| 1433 |
source=self) |
| 1434 |
|
| 1435 |
def add_child_handler(self, pid, callback, *args): |
| 1436 |
loop = events.get_running_loop() |
| 1437 |
thread = threading.Thread(target=self._do_waitpid, |
| 1438 |
name=f"asyncio-waitpid-{next(self._pid_counter)}", |
| 1439 |
args=(loop, pid, callback, args), |
| 1440 |
daemon=True) |
| 1441 |
self._threads[pid] = thread |
| 1442 |
thread.start() |
| 1443 |
|
| 1444 |
def remove_child_handler(self, pid): |
| 1445 |
# asyncio never calls remove_child_handler() !!! |
| 1446 |
# The method is no-op but is implemented because |
| 1447 |
# abstract base classes require it. |
| 1448 |
return True |
| 1449 |
|
| 1450 |
def attach_loop(self, loop): |
| 1451 |
pass |
| 1452 |
|
| 1453 |
def _do_waitpid(self, loop, expected_pid, callback, args): |
| 1454 |
assert expected_pid > 0 |
| 1455 |
|
| 1456 |
try: |
| 1457 |
pid, status = os.waitpid(expected_pid, 0) |
| 1458 |
except ChildProcessError: |
| 1459 |
# The child process is already reaped |
| 1460 |
# (may happen if waitpid() is called elsewhere). |
| 1461 |
pid = expected_pid |
| 1462 |
returncode = 255 |
| 1463 |
logger.warning( |
| 1464 |
"Unknown child process pid %d, will report returncode 255", |
| 1465 |
pid) |
| 1466 |
else: |
| 1467 |
returncode = waitstatus_to_exitcode(status) |
| 1468 |
if loop.get_debug(): |
| 1469 |
logger.debug('process %s exited with returncode %s', |
| 1470 |
expected_pid, returncode) |
| 1471 |
|
| 1472 |
if loop.is_closed(): |
| 1473 |
logger.warning("Loop %r that handles pid %r is closed", loop, pid) |
| 1474 |
else: |
| 1475 |
loop.call_soon_threadsafe(callback, pid, returncode, *args) |
| 1476 |
|
| 1477 |
self._threads.pop(expected_pid) |
| 1478 |
|
| 1479 |
def can_use_pidfd(): |
| 1480 |
if not hasattr(os, 'pidfd_open'): |
| 1481 |
return False |
| 1482 |
try: |
| 1483 |
pid = os.getpid() |
| 1484 |
os.close(os.pidfd_open(pid, 0)) |
| 1485 |
except OSError: |
| 1486 |
# blocked by security policy like SECCOMP |
| 1487 |
return False |
| 1488 |
return True |
| 1489 |
|
| 1490 |
|
| 1491 |
class _UnixDefaultEventLoopPolicy(events.BaseDefaultEventLoopPolicy): |
| 1492 |
"""UNIX event loop policy with a watcher for child processes.""" |
| 1493 |
_loop_factory = _UnixSelectorEventLoop |
| 1494 |
|
| 1495 |
def __init__(self): |
| 1496 |
super().__init__() |
| 1497 |
self._watcher = None |
| 1498 |
|
| 1499 |
def _init_watcher(self): |
| 1500 |
with events._lock: |
| 1501 |
if self._watcher is None: # pragma: no branch |
| 1502 |
if can_use_pidfd(): |
| 1503 |
self._watcher = PidfdChildWatcher() |
| 1504 |
else: |
| 1505 |
self._watcher = ThreadedChildWatcher() |
| 1506 |
|
| 1507 |
def set_event_loop(self, loop): |
| 1508 |
"""Set the event loop. |
| 1509 |
|
| 1510 |
As a side effect, if a child watcher was set before, then calling |
| 1511 |
.set_event_loop() from the main thread will call .attach_loop(loop) on |
| 1512 |
the child watcher. |
| 1513 |
""" |
| 1514 |
|
| 1515 |
super().set_event_loop(loop) |
| 1516 |
|
| 1517 |
if (self._watcher is not None and |
| 1518 |
threading.current_thread() is threading.main_thread()): |
| 1519 |
self._watcher.attach_loop(loop) |
| 1520 |
|
| 1521 |
def get_child_watcher(self): |
| 1522 |
"""Get the watcher for child processes. |
| 1523 |
|
| 1524 |
If not yet set, a ThreadedChildWatcher object is automatically created. |
| 1525 |
""" |
| 1526 |
if self._watcher is None: |
| 1527 |
self._init_watcher() |
| 1528 |
|
| 1529 |
warnings._deprecated("get_child_watcher", |
| 1530 |
"{name!r} is deprecated as of Python 3.12 and will be " |
| 1531 |
"removed in Python {remove}.", remove=(3, 14)) |
| 1532 |
return self._watcher |
| 1533 |
|
| 1534 |
def set_child_watcher(self, watcher): |
| 1535 |
"""Set the watcher for child processes.""" |
| 1536 |
|
| 1537 |
assert watcher is None or isinstance(watcher, AbstractChildWatcher) |
| 1538 |
|
| 1539 |
if self._watcher is not None: |
| 1540 |
self._watcher.close() |
| 1541 |
|
| 1542 |
self._watcher = watcher |
| 1543 |
warnings._deprecated("set_child_watcher", |
| 1544 |
"{name!r} is deprecated as of Python 3.12 and will be " |
| 1545 |
"removed in Python {remove}.", remove=(3, 14)) |
| 1546 |
|
| 1547 |
|
| 1548 |
SelectorEventLoop = _UnixSelectorEventLoop |
| 1549 |
DefaultEventLoopPolicy = _UnixDefaultEventLoopPolicy |
| 1550 |
EventLoop = SelectorEventLoop |