| 1 |
"""Selector event loop for Unix with signal handling.""" |
| 2 |
|
| 3 |
import errno |
| 4 |
import io |
| 5 |
import itertools |
| 6 |
import os |
| 7 |
import selectors |
| 8 |
import signal |
| 9 |
import socket |
| 10 |
import stat |
| 11 |
import subprocess |
| 12 |
import sys |
| 13 |
import threading |
| 14 |
import warnings |
| 15 |
|
| 16 |
from . import base_events |
| 17 |
from . import base_subprocess |
| 18 |
from . import constants |
| 19 |
from . import coroutines |
| 20 |
from . import events |
| 21 |
from . import exceptions |
| 22 |
from . import futures |
| 23 |
from . import selector_events |
| 24 |
from . import tasks |
| 25 |
from . import transports |
| 26 |
from .log import logger |
| 27 |
|
| 28 |
|
| 29 |
__all__ = ( |
| 30 |
'SelectorEventLoop', |
| 31 |
'EventLoop', |
| 32 |
) |
| 33 |
|
| 34 |
|
| 35 |
if sys.platform == 'win32': # pragma: no cover |
| 36 |
raise ImportError('Signals are not really supported on Windows') |
| 37 |
|
| 38 |
|
| 39 |
def _sighandler_noop(signum, frame): |
| 40 |
"""Dummy signal handler.""" |
| 41 |
pass |
| 42 |
|
| 43 |
|
| 44 |
def waitstatus_to_exitcode(status): |
| 45 |
try: |
| 46 |
return os.waitstatus_to_exitcode(status) |
| 47 |
except ValueError: |
| 48 |
# The child exited, but we don't understand its status. |
| 49 |
# This shouldn't happen, but if it does, let's just |
| 50 |
# return that status; perhaps that helps debug it. |
| 51 |
return status |
| 52 |
|
| 53 |
|
| 54 |
class _UnixSelectorEventLoop(selector_events.BaseSelectorEventLoop): |
| 55 |
"""Unix event loop. |
| 56 |
|
| 57 |
Adds signal handling and UNIX Domain Socket support to |
| 58 |
SelectorEventLoop. |
| 59 |
""" |
| 60 |
|
| 61 |
def __init__(self, selector=None): |
| 62 |
super().__init__(selector) |
| 63 |
self._signal_handlers = {} |
| 64 |
self._unix_server_sockets = {} |
| 65 |
if can_use_pidfd(): |
| 66 |
self._watcher = _PidfdChildWatcher() |
| 67 |
else: |
| 68 |
self._watcher = _ThreadedChildWatcher() |
| 69 |
|
| 70 |
def close(self): |
| 71 |
super().close() |
| 72 |
if not sys.is_finalizing(): |
| 73 |
for sig in list(self._signal_handlers): |
| 74 |
self.remove_signal_handler(sig) |
| 75 |
else: |
| 76 |
if self._signal_handlers: |
| 77 |
warnings.warn(f"Closing the loop {self!r} " |
| 78 |
f"on interpreter shutdown " |
| 79 |
f"stage, skipping signal handlers removal", |
| 80 |
ResourceWarning, |
| 81 |
source=self) |
| 82 |
self._signal_handlers.clear() |
| 83 |
|
| 84 |
def _process_self_data(self, data): |
| 85 |
for signum in data: |
| 86 |
if not signum: |
| 87 |
# ignore null bytes written by _write_to_self() |
| 88 |
continue |
| 89 |
self._handle_signal(signum) |
| 90 |
|
| 91 |
def add_signal_handler(self, sig, callback, *args): |
| 92 |
"""Add a handler for a signal. UNIX only. |
| 93 |
|
| 94 |
Raise ValueError if the signal number is invalid or uncatchable. |
| 95 |
Raise RuntimeError if there is a problem setting up the handler. |
| 96 |
""" |
| 97 |
if (coroutines.iscoroutine(callback) or |
| 98 |
coroutines._iscoroutinefunction(callback)): |
| 99 |
raise TypeError("coroutines cannot be used " |
| 100 |
"with add_signal_handler()") |
| 101 |
self._check_signal(sig) |
| 102 |
self._check_closed() |
| 103 |
try: |
| 104 |
# set_wakeup_fd() raises ValueError if this is not the |
| 105 |
# main thread. By calling it early we ensure that an |
| 106 |
# event loop running in another thread cannot add a signal |
| 107 |
# handler. |
| 108 |
signal.set_wakeup_fd(self._csock.fileno()) |
| 109 |
except (ValueError, OSError) as exc: |
| 110 |
raise RuntimeError(str(exc)) |
| 111 |
|
| 112 |
handle = events.Handle(callback, args, self, None) |
| 113 |
self._signal_handlers[sig] = handle |
| 114 |
|
| 115 |
try: |
| 116 |
# Register a dummy signal handler to ask Python to write the signal |
| 117 |
# number in the wakeup file descriptor. _process_self_data() will |
| 118 |
# read signal numbers from this file descriptor to handle signals. |
| 119 |
signal.signal(sig, _sighandler_noop) |
| 120 |
|
| 121 |
# Set SA_RESTART to limit EINTR occurrences. |
| 122 |
signal.siginterrupt(sig, False) |
| 123 |
except OSError as exc: |
| 124 |
del self._signal_handlers[sig] |
| 125 |
if not self._signal_handlers: |
| 126 |
try: |
| 127 |
signal.set_wakeup_fd(-1) |
| 128 |
except (ValueError, OSError) as nexc: |
| 129 |
logger.info('set_wakeup_fd(-1) failed: %s', nexc) |
| 130 |
|
| 131 |
if exc.errno == errno.EINVAL: |
| 132 |
raise RuntimeError(f'sig {sig} cannot be caught') |
| 133 |
else: |
| 134 |
raise |
| 135 |
|
| 136 |
def _handle_signal(self, sig): |
| 137 |
"""Internal helper that is the actual signal handler.""" |
| 138 |
handle = self._signal_handlers.get(sig) |
| 139 |
if handle is None: |
| 140 |
return # Assume it's some race condition. |
| 141 |
if handle._cancelled: |
| 142 |
self.remove_signal_handler(sig) # Remove it properly. |
| 143 |
else: |
| 144 |
self._add_callback_signalsafe(handle) |
| 145 |
|
| 146 |
def remove_signal_handler(self, sig): |
| 147 |
"""Remove a handler for a signal. UNIX only. |
| 148 |
|
| 149 |
Return True if a signal handler was removed, False if not. |
| 150 |
""" |
| 151 |
self._check_signal(sig) |
| 152 |
try: |
| 153 |
del self._signal_handlers[sig] |
| 154 |
except KeyError: |
| 155 |
return False |
| 156 |
|
| 157 |
if sig == signal.SIGINT: |
| 158 |
handler = signal.default_int_handler |
| 159 |
else: |
| 160 |
handler = signal.SIG_DFL |
| 161 |
|
| 162 |
try: |
| 163 |
signal.signal(sig, handler) |
| 164 |
except OSError as exc: |
| 165 |
if exc.errno == errno.EINVAL: |
| 166 |
raise RuntimeError(f'sig {sig} cannot be caught') |
| 167 |
else: |
| 168 |
raise |
| 169 |
|
| 170 |
if not self._signal_handlers: |
| 171 |
try: |
| 172 |
signal.set_wakeup_fd(-1) |
| 173 |
except (ValueError, OSError) as exc: |
| 174 |
logger.info('set_wakeup_fd(-1) failed: %s', exc) |
| 175 |
|
| 176 |
return True |
| 177 |
|
| 178 |
def _check_signal(self, sig): |
| 179 |
"""Internal helper to validate a signal. |
| 180 |
|
| 181 |
Raise ValueError if the signal number is invalid or uncatchable. |
| 182 |
Raise RuntimeError if there is a problem setting up the handler. |
| 183 |
""" |
| 184 |
if not isinstance(sig, int): |
| 185 |
raise TypeError(f'sig must be an int, not {sig!r}') |
| 186 |
|
| 187 |
if sig not in signal.valid_signals(): |
| 188 |
raise ValueError(f'invalid signal number {sig}') |
| 189 |
|
| 190 |
def _make_read_pipe_transport(self, pipe, protocol, waiter=None, |
| 191 |
extra=None): |
| 192 |
return _UnixReadPipeTransport(self, pipe, protocol, waiter, extra) |
| 193 |
|
| 194 |
def _make_write_pipe_transport(self, pipe, protocol, waiter=None, |
| 195 |
extra=None): |
| 196 |
return _UnixWritePipeTransport(self, pipe, protocol, waiter, extra) |
| 197 |
|
| 198 |
async def _make_subprocess_transport(self, protocol, args, shell, |
| 199 |
stdin, stdout, stderr, bufsize, |
| 200 |
extra=None, **kwargs): |
| 201 |
watcher = self._watcher |
| 202 |
waiter = self.create_future() |
| 203 |
transp = _UnixSubprocessTransport(self, protocol, args, shell, |
| 204 |
stdin, stdout, stderr, bufsize, |
| 205 |
waiter=waiter, extra=extra, |
| 206 |
**kwargs) |
| 207 |
watcher.add_child_handler(transp.get_pid(), |
| 208 |
self._child_watcher_callback, transp) |
| 209 |
try: |
| 210 |
await waiter |
| 211 |
except (SystemExit, KeyboardInterrupt): |
| 212 |
raise |
| 213 |
except BaseException: |
| 214 |
transp.close() |
| 215 |
await tasks.shield(transp._wait()) |
| 216 |
raise |
| 217 |
|
| 218 |
return transp |
| 219 |
|
| 220 |
def _child_watcher_callback(self, pid, returncode, transp): |
| 221 |
transp._process_exited(returncode) |
| 222 |
|
| 223 |
async def create_unix_connection( |
| 224 |
self, protocol_factory, path=None, *, |
| 225 |
ssl=None, sock=None, |
| 226 |
server_hostname=None, |
| 227 |
ssl_handshake_timeout=None, |
| 228 |
ssl_shutdown_timeout=None): |
| 229 |
assert server_hostname is None or isinstance(server_hostname, str) |
| 230 |
if ssl: |
| 231 |
if server_hostname is None: |
| 232 |
raise ValueError( |
| 233 |
'you have to pass server_hostname when using ssl') |
| 234 |
else: |
| 235 |
if server_hostname is not None: |
| 236 |
raise ValueError('server_hostname is only meaningful with ssl') |
| 237 |
if ssl_handshake_timeout is not None: |
| 238 |
raise ValueError( |
| 239 |
'ssl_handshake_timeout is only meaningful with ssl') |
| 240 |
if ssl_shutdown_timeout is not None: |
| 241 |
raise ValueError( |
| 242 |
'ssl_shutdown_timeout is only meaningful with ssl') |
| 243 |
|
| 244 |
if path is not None: |
| 245 |
if sock is not None: |
| 246 |
raise ValueError( |
| 247 |
'path and sock can not be specified at the same time') |
| 248 |
|
| 249 |
path = os.fspath(path) |
| 250 |
sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM, 0) |
| 251 |
try: |
| 252 |
sock.setblocking(False) |
| 253 |
await self.sock_connect(sock, path) |
| 254 |
except: |
| 255 |
sock.close() |
| 256 |
raise |
| 257 |
|
| 258 |
else: |
| 259 |
if sock is None: |
| 260 |
raise ValueError('no path and sock were specified') |
| 261 |
if (sock.family != socket.AF_UNIX or |
| 262 |
sock.type != socket.SOCK_STREAM): |
| 263 |
raise ValueError( |
| 264 |
f'A UNIX Domain Stream Socket was expected, got {sock!r}') |
| 265 |
sock.setblocking(False) |
| 266 |
|
| 267 |
transport, protocol = await self._create_connection_transport( |
| 268 |
sock, protocol_factory, ssl, server_hostname, |
| 269 |
ssl_handshake_timeout=ssl_handshake_timeout, |
| 270 |
ssl_shutdown_timeout=ssl_shutdown_timeout) |
| 271 |
return transport, protocol |
| 272 |
|
| 273 |
async def create_unix_server( |
| 274 |
self, protocol_factory, path=None, *, |
| 275 |
sock=None, backlog=100, ssl=None, |
| 276 |
ssl_handshake_timeout=None, |
| 277 |
ssl_shutdown_timeout=None, |
| 278 |
start_serving=True, cleanup_socket=True): |
| 279 |
if isinstance(ssl, bool): |
| 280 |
raise TypeError('ssl argument must be an SSLContext or None') |
| 281 |
|
| 282 |
if ssl_handshake_timeout is not None and not ssl: |
| 283 |
raise ValueError( |
| 284 |
'ssl_handshake_timeout is only meaningful with ssl') |
| 285 |
|
| 286 |
if ssl_shutdown_timeout is not None and not ssl: |
| 287 |
raise ValueError( |
| 288 |
'ssl_shutdown_timeout is only meaningful with ssl') |
| 289 |
|
| 290 |
if path is not None: |
| 291 |
if sock is not None: |
| 292 |
raise ValueError( |
| 293 |
'path and sock can not be specified at the same time') |
| 294 |
|
| 295 |
path = os.fspath(path) |
| 296 |
sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) |
| 297 |
|
| 298 |
# Check for abstract socket. `str` and `bytes` paths are supported. |
| 299 |
if path[0] not in (0, '\x00'): |
| 300 |
try: |
| 301 |
if stat.S_ISSOCK(os.stat(path).st_mode): |
| 302 |
os.remove(path) |
| 303 |
except FileNotFoundError: |
| 304 |
pass |
| 305 |
except OSError as err: |
| 306 |
# Directory may have permissions only to create socket. |
| 307 |
logger.error('Unable to check or remove stale UNIX socket ' |
| 308 |
'%r: %r', path, err) |
| 309 |
|
| 310 |
try: |
| 311 |
sock.bind(path) |
| 312 |
except OSError as exc: |
| 313 |
sock.close() |
| 314 |
if exc.errno == errno.EADDRINUSE: |
| 315 |
# Let's improve the error message by adding |
| 316 |
# with what exact address it occurs. |
| 317 |
msg = f'Address {path!r} is already in use' |
| 318 |
raise OSError(errno.EADDRINUSE, msg) from None |
| 319 |
else: |
| 320 |
raise |
| 321 |
except: |
| 322 |
sock.close() |
| 323 |
raise |
| 324 |
else: |
| 325 |
if sock is None: |
| 326 |
raise ValueError( |
| 327 |
'path was not specified, and no sock specified') |
| 328 |
|
| 329 |
if (sock.family != socket.AF_UNIX or |
| 330 |
sock.type != socket.SOCK_STREAM): |
| 331 |
raise ValueError( |
| 332 |
f'A UNIX Domain Stream Socket was expected, got {sock!r}') |
| 333 |
|
| 334 |
if cleanup_socket: |
| 335 |
path = sock.getsockname() |
| 336 |
# Check for abstract socket. `str` and `bytes` paths are supported. |
| 337 |
if path[0] not in (0, '\x00'): |
| 338 |
try: |
| 339 |
self._unix_server_sockets[sock] = os.stat(path).st_ino |
| 340 |
except FileNotFoundError: |
| 341 |
pass |
| 342 |
|
| 343 |
sock.setblocking(False) |
| 344 |
server = base_events.Server(self, [sock], protocol_factory, |
| 345 |
ssl, backlog, ssl_handshake_timeout, |
| 346 |
ssl_shutdown_timeout) |
| 347 |
if start_serving: |
| 348 |
server._start_serving() |
| 349 |
# Skip one loop iteration so that all 'loop.add_reader' |
| 350 |
# go through. |
| 351 |
await tasks.sleep(0) |
| 352 |
|
| 353 |
return server |
| 354 |
|
| 355 |
async def _sock_sendfile_native(self, sock, file, offset, count): |
| 356 |
try: |
| 357 |
os.sendfile |
| 358 |
except AttributeError: |
| 359 |
raise exceptions.SendfileNotAvailableError( |
| 360 |
"os.sendfile() is not available") |
| 361 |
try: |
| 362 |
fileno = file.fileno() |
| 363 |
except (AttributeError, io.UnsupportedOperation) as err: |
| 364 |
raise exceptions.SendfileNotAvailableError("not a regular file") |
| 365 |
try: |
| 366 |
fsize = os.fstat(fileno).st_size |
| 367 |
except OSError: |
| 368 |
raise exceptions.SendfileNotAvailableError("not a regular file") |
| 369 |
blocksize = count if count else fsize |
| 370 |
if not blocksize: |
| 371 |
return 0 # empty file |
| 372 |
|
| 373 |
fut = self.create_future() |
| 374 |
self._sock_sendfile_native_impl(fut, None, sock, fileno, |
| 375 |
offset, count, blocksize, 0) |
| 376 |
return await fut |
| 377 |
|
| 378 |
def _sock_sendfile_native_impl(self, fut, registered_fd, sock, fileno, |
| 379 |
offset, count, blocksize, total_sent): |
| 380 |
fd = sock.fileno() |
| 381 |
if registered_fd is not None: |
| 382 |
# Remove the callback early. It should be rare that the |
| 383 |
# selector says the fd is ready but the call still returns |
| 384 |
# EAGAIN, and I am willing to take a hit in that case in |
| 385 |
# order to simplify the common case. |
| 386 |
self.remove_writer(registered_fd) |
| 387 |
if fut.cancelled(): |
| 388 |
self._sock_sendfile_update_filepos(fileno, offset) |
| 389 |
return |
| 390 |
if count: |
| 391 |
blocksize = count - total_sent |
| 392 |
if blocksize <= 0: |
| 393 |
self._sock_sendfile_update_filepos(fileno, offset) |
| 394 |
fut.set_result(total_sent) |
| 395 |
return |
| 396 |
|
| 397 |
# On 32-bit architectures truncate to 1GiB to avoid OverflowError |
| 398 |
blocksize = min(blocksize, sys.maxsize//2 + 1) |
| 399 |
|
| 400 |
try: |
| 401 |
sent = os.sendfile(fd, fileno, offset, blocksize) |
| 402 |
except (BlockingIOError, InterruptedError): |
| 403 |
if registered_fd is None: |
| 404 |
self._sock_add_cancellation_callback(fut, sock) |
| 405 |
self.add_writer(fd, self._sock_sendfile_native_impl, fut, |
| 406 |
fd, sock, fileno, |
| 407 |
offset, count, blocksize, total_sent) |
| 408 |
except OSError as exc: |
| 409 |
if (registered_fd is not None and |
| 410 |
exc.errno == errno.ENOTCONN and |
| 411 |
type(exc) is not ConnectionError): |
| 412 |
# If we have an ENOTCONN and this isn't a first call to |
| 413 |
# sendfile(), i.e. the connection was closed in the middle |
| 414 |
# of the operation, normalize the error to ConnectionError |
| 415 |
# to make it consistent across all Posix systems. |
| 416 |
new_exc = ConnectionError( |
| 417 |
"socket is not connected", errno.ENOTCONN) |
| 418 |
new_exc.__cause__ = exc |
| 419 |
exc = new_exc |
| 420 |
if total_sent == 0: |
| 421 |
# We can get here for different reasons, the main |
| 422 |
# one being 'file' is not a regular mmap(2)-like |
| 423 |
# file, in which case we'll fall back on using |
| 424 |
# plain send(). |
| 425 |
err = exceptions.SendfileNotAvailableError( |
| 426 |
"os.sendfile call failed") |
| 427 |
self._sock_sendfile_update_filepos(fileno, offset) |
| 428 |
fut.set_exception(err) |
| 429 |
else: |
| 430 |
self._sock_sendfile_update_filepos(fileno, offset) |
| 431 |
fut.set_exception(exc) |
| 432 |
except (SystemExit, KeyboardInterrupt): |
| 433 |
raise |
| 434 |
except BaseException as exc: |
| 435 |
self._sock_sendfile_update_filepos(fileno, offset) |
| 436 |
fut.set_exception(exc) |
| 437 |
else: |
| 438 |
if sent == 0: |
| 439 |
# EOF |
| 440 |
self._sock_sendfile_update_filepos(fileno, offset) |
| 441 |
fut.set_result(total_sent) |
| 442 |
else: |
| 443 |
offset += sent |
| 444 |
total_sent += sent |
| 445 |
if registered_fd is None: |
| 446 |
self._sock_add_cancellation_callback(fut, sock) |
| 447 |
self.add_writer(fd, self._sock_sendfile_native_impl, fut, |
| 448 |
fd, sock, fileno, |
| 449 |
offset, count, blocksize, total_sent) |
| 450 |
|
| 451 |
def _sock_sendfile_update_filepos(self, fileno, offset): |
| 452 |
# After this helper runs, the source fd's lseek pointer is at offset." |
| 453 |
os.lseek(fileno, offset, os.SEEK_SET) |
| 454 |
|
| 455 |
def _sock_add_cancellation_callback(self, fut, sock): |
| 456 |
def cb(fut): |
| 457 |
if fut.cancelled(): |
| 458 |
fd = sock.fileno() |
| 459 |
if fd != -1: |
| 460 |
self.remove_writer(fd) |
| 461 |
fut.add_done_callback(cb) |
| 462 |
|
| 463 |
def _stop_serving(self, sock): |
| 464 |
# Is this a unix socket that needs cleanup? |
| 465 |
if sock in self._unix_server_sockets: |
| 466 |
path = sock.getsockname() |
| 467 |
else: |
| 468 |
path = None |
| 469 |
|
| 470 |
super()._stop_serving(sock) |
| 471 |
|
| 472 |
if path is not None: |
| 473 |
prev_ino = self._unix_server_sockets[sock] |
| 474 |
del self._unix_server_sockets[sock] |
| 475 |
try: |
| 476 |
if os.stat(path).st_ino == prev_ino: |
| 477 |
os.unlink(path) |
| 478 |
except FileNotFoundError: |
| 479 |
pass |
| 480 |
except OSError as err: |
| 481 |
logger.error('Unable to clean up listening UNIX socket ' |
| 482 |
'%r: %r', path, err) |
| 483 |
|
| 484 |
|
| 485 |
class _UnixReadPipeTransport(transports.ReadTransport): |
| 486 |
|
| 487 |
max_size = 256 * 1024 # max bytes we read in one event loop iteration |
| 488 |
|
| 489 |
def __init__(self, loop, pipe, protocol, waiter=None, extra=None): |
| 490 |
super().__init__(extra) |
| 491 |
self._extra['pipe'] = pipe |
| 492 |
self._loop = loop |
| 493 |
self._pipe = pipe |
| 494 |
self._fileno = pipe.fileno() |
| 495 |
self._protocol = protocol |
| 496 |
self._closing = False |
| 497 |
self._paused = False |
| 498 |
|
| 499 |
mode = os.fstat(self._fileno).st_mode |
| 500 |
if not (stat.S_ISFIFO(mode) or |
| 501 |
stat.S_ISSOCK(mode) or |
| 502 |
stat.S_ISCHR(mode)): |
| 503 |
self._pipe = None |
| 504 |
self._fileno = None |
| 505 |
self._protocol = None |
| 506 |
raise ValueError("Pipe transport is for pipes/sockets only.") |
| 507 |
|
| 508 |
os.set_blocking(self._fileno, False) |
| 509 |
|
| 510 |
self._loop.call_soon(self._protocol.connection_made, self) |
| 511 |
# only start reading when connection_made() has been called |
| 512 |
self._loop.call_soon(self._add_reader, |
| 513 |
self._fileno, self._read_ready) |
| 514 |
if waiter is not None: |
| 515 |
# only wake up the waiter when connection_made() has been called |
| 516 |
self._loop.call_soon(futures._set_result_unless_cancelled, |
| 517 |
waiter, None) |
| 518 |
|
| 519 |
def _add_reader(self, fd, callback): |
| 520 |
if not self.is_reading(): |
| 521 |
return |
| 522 |
self._loop._add_reader(fd, callback) |
| 523 |
|
| 524 |
def is_reading(self): |
| 525 |
return not self._paused and not self._closing |
| 526 |
|
| 527 |
def __repr__(self): |
| 528 |
info = [self.__class__.__name__] |
| 529 |
if self._pipe is None: |
| 530 |
info.append('closed') |
| 531 |
elif self._closing: |
| 532 |
info.append('closing') |
| 533 |
info.append(f'fd={self._fileno}') |
| 534 |
selector = getattr(self._loop, '_selector', None) |
| 535 |
if self._pipe is not None and selector is not None: |
| 536 |
polling = selector_events._test_selector_event( |
| 537 |
selector, self._fileno, selectors.EVENT_READ) |
| 538 |
if polling: |
| 539 |
info.append('polling') |
| 540 |
else: |
| 541 |
info.append('idle') |
| 542 |
elif self._pipe is not None: |
| 543 |
info.append('open') |
| 544 |
else: |
| 545 |
info.append('closed') |
| 546 |
return '<{}>'.format(' '.join(info)) |
| 547 |
|
| 548 |
def _read_ready(self): |
| 549 |
try: |
| 550 |
data = os.read(self._fileno, self.max_size) |
| 551 |
except (BlockingIOError, InterruptedError): |
| 552 |
pass |
| 553 |
except OSError as exc: |
| 554 |
self._fatal_error(exc, 'Fatal read error on pipe transport') |
| 555 |
else: |
| 556 |
if data: |
| 557 |
self._protocol.data_received(data) |
| 558 |
else: |
| 559 |
if self._loop.get_debug(): |
| 560 |
logger.info("%r was closed by peer", self) |
| 561 |
self._closing = True |
| 562 |
self._loop._remove_reader(self._fileno) |
| 563 |
self._loop.call_soon(self._protocol.eof_received) |
| 564 |
self._loop.call_soon(self._call_connection_lost, None) |
| 565 |
|
| 566 |
def pause_reading(self): |
| 567 |
if not self.is_reading(): |
| 568 |
return |
| 569 |
self._paused = True |
| 570 |
self._loop._remove_reader(self._fileno) |
| 571 |
if self._loop.get_debug(): |
| 572 |
logger.debug("%r pauses reading", self) |
| 573 |
|
| 574 |
def resume_reading(self): |
| 575 |
if self._closing or not self._paused: |
| 576 |
return |
| 577 |
self._paused = False |
| 578 |
self._loop._add_reader(self._fileno, self._read_ready) |
| 579 |
if self._loop.get_debug(): |
| 580 |
logger.debug("%r resumes reading", self) |
| 581 |
|
| 582 |
def set_protocol(self, protocol): |
| 583 |
self._protocol = protocol |
| 584 |
|
| 585 |
def get_protocol(self): |
| 586 |
return self._protocol |
| 587 |
|
| 588 |
def is_closing(self): |
| 589 |
return self._closing |
| 590 |
|
| 591 |
def close(self): |
| 592 |
if not self._closing: |
| 593 |
self._close(None) |
| 594 |
|
| 595 |
def __del__(self, _warn=warnings.warn): |
| 596 |
if self._pipe is not None: |
| 597 |
_warn(f"unclosed transport {self!r}", ResourceWarning, source=self) |
| 598 |
self._pipe.close() |
| 599 |
|
| 600 |
def _fatal_error(self, exc, message='Fatal error on pipe transport'): |
| 601 |
# should be called by exception handler only |
| 602 |
if (isinstance(exc, OSError) and exc.errno == errno.EIO): |
| 603 |
if self._loop.get_debug(): |
| 604 |
logger.debug("%r: %s", self, message, exc_info=True) |
| 605 |
else: |
| 606 |
self._loop.call_exception_handler({ |
| 607 |
'message': message, |
| 608 |
'exception': exc, |
| 609 |
'transport': self, |
| 610 |
'protocol': self._protocol, |
| 611 |
}) |
| 612 |
self._close(exc) |
| 613 |
|
| 614 |
def _close(self, exc): |
| 615 |
self._closing = True |
| 616 |
self._loop._remove_reader(self._fileno) |
| 617 |
self._loop.call_soon(self._call_connection_lost, exc) |
| 618 |
|
| 619 |
def _call_connection_lost(self, exc): |
| 620 |
try: |
| 621 |
self._protocol.connection_lost(exc) |
| 622 |
finally: |
| 623 |
self._pipe.close() |
| 624 |
self._pipe = None |
| 625 |
self._protocol = None |
| 626 |
self._loop = None |
| 627 |
|
| 628 |
|
| 629 |
class _UnixWritePipeTransport(transports._FlowControlMixin, |
| 630 |
transports.WriteTransport): |
| 631 |
|
| 632 |
def __init__(self, loop, pipe, protocol, waiter=None, extra=None): |
| 633 |
super().__init__(extra, loop) |
| 634 |
self._extra['pipe'] = pipe |
| 635 |
self._pipe = pipe |
| 636 |
self._fileno = pipe.fileno() |
| 637 |
self._protocol = protocol |
| 638 |
self._buffer = bytearray() |
| 639 |
self._conn_lost = 0 |
| 640 |
self._closing = False # Set when close() or write_eof() called. |
| 641 |
|
| 642 |
pipe_stat = os.fstat(self._fileno) |
| 643 |
mode = pipe_stat.st_mode |
| 644 |
is_char = stat.S_ISCHR(mode) |
| 645 |
is_fifo = stat.S_ISFIFO(mode) |
| 646 |
is_socket = stat.S_ISSOCK(mode) |
| 647 |
if not (is_char or is_fifo or is_socket): |
| 648 |
self._pipe = None |
| 649 |
self._fileno = None |
| 650 |
self._protocol = None |
| 651 |
raise ValueError("Pipe transport is only for " |
| 652 |
"pipes, sockets and character devices") |
| 653 |
|
| 654 |
os.set_blocking(self._fileno, False) |
| 655 |
self._loop.call_soon(self._protocol.connection_made, self) |
| 656 |
|
| 657 |
# On AIX, the reader trick (to be notified when the read end of the |
| 658 |
# socket is closed) only works for sockets. On other platforms it |
| 659 |
# works for pipes and sockets. (Exception: OS X 10.4? Issue #19294.) |
| 660 |
# On macOS and Solaris, the trick misfires for named FIFOs (but not for |
| 661 |
# pipes created with os.pipe(), which have st_nlink == 0): the write |
| 662 |
# end polls as readable whenever unread data sits in the FIFO, and no |
| 663 |
# event is delivered when the read end is closed, so it can only |
| 664 |
# ever report a false disconnection (gh-145030). The same XNU |
| 665 |
# behaviour applies on iOS/tvOS/watchOS (sys.platform is not |
| 666 |
# "darwin" there). |
| 667 |
is_named_fifo_without_close_event = ( |
| 668 |
sys.platform in {"darwin", "ios", "tvos", "watchos", "sunos5"} |
| 669 |
and is_fifo and pipe_stat.st_nlink > 0) |
| 670 |
if is_socket or (is_fifo |
| 671 |
and not sys.platform.startswith("aix") |
| 672 |
and not is_named_fifo_without_close_event): |
| 673 |
# only start reading when connection_made() has been called |
| 674 |
self._loop.call_soon(self._loop._add_reader, |
| 675 |
self._fileno, self._read_ready) |
| 676 |
|
| 677 |
if waiter is not None: |
| 678 |
# only wake up the waiter when connection_made() has been called |
| 679 |
self._loop.call_soon(futures._set_result_unless_cancelled, |
| 680 |
waiter, None) |
| 681 |
|
| 682 |
def __repr__(self): |
| 683 |
info = [self.__class__.__name__] |
| 684 |
if self._pipe is None: |
| 685 |
info.append('closed') |
| 686 |
elif self._closing: |
| 687 |
info.append('closing') |
| 688 |
info.append(f'fd={self._fileno}') |
| 689 |
selector = getattr(self._loop, '_selector', None) |
| 690 |
if self._pipe is not None and selector is not None: |
| 691 |
polling = selector_events._test_selector_event( |
| 692 |
selector, self._fileno, selectors.EVENT_WRITE) |
| 693 |
if polling: |
| 694 |
info.append('polling') |
| 695 |
else: |
| 696 |
info.append('idle') |
| 697 |
|
| 698 |
bufsize = self.get_write_buffer_size() |
| 699 |
info.append(f'bufsize={bufsize}') |
| 700 |
elif self._pipe is not None: |
| 701 |
info.append('open') |
| 702 |
else: |
| 703 |
info.append('closed') |
| 704 |
return '<{}>'.format(' '.join(info)) |
| 705 |
|
| 706 |
def get_write_buffer_size(self): |
| 707 |
return len(self._buffer) |
| 708 |
|
| 709 |
def _read_ready(self): |
| 710 |
# Pipe was closed by peer. |
| 711 |
if self._loop.get_debug(): |
| 712 |
logger.info("%r was closed by peer", self) |
| 713 |
if self._buffer: |
| 714 |
self._close(BrokenPipeError()) |
| 715 |
else: |
| 716 |
self._close() |
| 717 |
|
| 718 |
def write(self, data): |
| 719 |
assert isinstance(data, (bytes, bytearray, memoryview)), repr(data) |
| 720 |
if isinstance(data, bytearray): |
| 721 |
data = memoryview(data) |
| 722 |
if not data: |
| 723 |
return |
| 724 |
|
| 725 |
if self._conn_lost or self._closing: |
| 726 |
if self._conn_lost >= constants.LOG_THRESHOLD_FOR_CONNLOST_WRITES: |
| 727 |
logger.warning('pipe closed by peer or ' |
| 728 |
'os.write(pipe, data) raised exception.') |
| 729 |
self._conn_lost += 1 |
| 730 |
return |
| 731 |
|
| 732 |
if not self._buffer: |
| 733 |
# Attempt to send it right away first. |
| 734 |
try: |
| 735 |
n = os.write(self._fileno, data) |
| 736 |
except (BlockingIOError, InterruptedError): |
| 737 |
n = 0 |
| 738 |
except (SystemExit, KeyboardInterrupt): |
| 739 |
raise |
| 740 |
except BaseException as exc: |
| 741 |
self._conn_lost += 1 |
| 742 |
self._fatal_error(exc, 'Fatal write error on pipe transport') |
| 743 |
return |
| 744 |
if n == len(data): |
| 745 |
return |
| 746 |
elif n > 0: |
| 747 |
data = memoryview(data)[n:] |
| 748 |
self._loop._add_writer(self._fileno, self._write_ready) |
| 749 |
|
| 750 |
self._buffer += data |
| 751 |
self._maybe_pause_protocol() |
| 752 |
|
| 753 |
def _write_ready(self): |
| 754 |
assert self._buffer, 'Data should not be empty' |
| 755 |
|
| 756 |
try: |
| 757 |
n = os.write(self._fileno, self._buffer) |
| 758 |
except (BlockingIOError, InterruptedError): |
| 759 |
pass |
| 760 |
except (SystemExit, KeyboardInterrupt): |
| 761 |
raise |
| 762 |
except BaseException as exc: |
| 763 |
self._buffer.clear() |
| 764 |
self._conn_lost += 1 |
| 765 |
# Remove writer here, _fatal_error() doesn't it |
| 766 |
# because _buffer is empty. |
| 767 |
self._loop._remove_writer(self._fileno) |
| 768 |
self._fatal_error(exc, 'Fatal write error on pipe transport') |
| 769 |
else: |
| 770 |
if n == len(self._buffer): |
| 771 |
self._buffer.clear() |
| 772 |
self._loop._remove_writer(self._fileno) |
| 773 |
self._maybe_resume_protocol() # May append to buffer. |
| 774 |
if self._closing: |
| 775 |
self._loop._remove_reader(self._fileno) |
| 776 |
self._call_connection_lost(None) |
| 777 |
return |
| 778 |
elif n > 0: |
| 779 |
del self._buffer[:n] |
| 780 |
|
| 781 |
def can_write_eof(self): |
| 782 |
return True |
| 783 |
|
| 784 |
def write_eof(self): |
| 785 |
if self._closing: |
| 786 |
return |
| 787 |
assert self._pipe |
| 788 |
self._closing = True |
| 789 |
if not self._buffer: |
| 790 |
self._loop._remove_reader(self._fileno) |
| 791 |
self._loop.call_soon(self._call_connection_lost, None) |
| 792 |
|
| 793 |
def set_protocol(self, protocol): |
| 794 |
self._protocol = protocol |
| 795 |
|
| 796 |
def get_protocol(self): |
| 797 |
return self._protocol |
| 798 |
|
| 799 |
def is_closing(self): |
| 800 |
return self._closing |
| 801 |
|
| 802 |
def close(self): |
| 803 |
if self._pipe is not None and not self._closing: |
| 804 |
# write_eof is all what we needed to close the write pipe |
| 805 |
self.write_eof() |
| 806 |
|
| 807 |
def __del__(self, _warn=warnings.warn): |
| 808 |
if self._pipe is not None: |
| 809 |
_warn(f"unclosed transport {self!r}", ResourceWarning, source=self) |
| 810 |
self._pipe.close() |
| 811 |
|
| 812 |
def abort(self): |
| 813 |
self._close(None) |
| 814 |
|
| 815 |
def _fatal_error(self, exc, message='Fatal error on pipe transport'): |
| 816 |
# should be called by exception handler only |
| 817 |
if isinstance(exc, OSError): |
| 818 |
if self._loop.get_debug(): |
| 819 |
logger.debug("%r: %s", self, message, exc_info=True) |
| 820 |
else: |
| 821 |
self._loop.call_exception_handler({ |
| 822 |
'message': message, |
| 823 |
'exception': exc, |
| 824 |
'transport': self, |
| 825 |
'protocol': self._protocol, |
| 826 |
}) |
| 827 |
self._close(exc) |
| 828 |
|
| 829 |
def _close(self, exc=None): |
| 830 |
self._closing = True |
| 831 |
if self._buffer: |
| 832 |
self._loop._remove_writer(self._fileno) |
| 833 |
self._buffer.clear() |
| 834 |
self._loop._remove_reader(self._fileno) |
| 835 |
self._loop.call_soon(self._call_connection_lost, exc) |
| 836 |
|
| 837 |
def _call_connection_lost(self, exc): |
| 838 |
try: |
| 839 |
self._protocol.connection_lost(exc) |
| 840 |
finally: |
| 841 |
self._pipe.close() |
| 842 |
self._pipe = None |
| 843 |
self._protocol = None |
| 844 |
self._loop = None |
| 845 |
|
| 846 |
|
| 847 |
class _UnixSubprocessTransport(base_subprocess.BaseSubprocessTransport): |
| 848 |
|
| 849 |
def _start(self, args, shell, stdin, stdout, stderr, bufsize, **kwargs): |
| 850 |
stdin_w = None |
| 851 |
if stdin == subprocess.PIPE and sys.platform.startswith('aix'): |
| 852 |
# Use a socket pair for stdin on AIX, since it does not |
| 853 |
# support selecting read events on the write end of a |
| 854 |
# socket (which we use in order to detect closing of the |
| 855 |
# other end). |
| 856 |
stdin, stdin_w = socket.socketpair() |
| 857 |
try: |
| 858 |
self._proc = subprocess.Popen( |
| 859 |
args, shell=shell, stdin=stdin, stdout=stdout, stderr=stderr, |
| 860 |
universal_newlines=False, bufsize=bufsize, **kwargs) |
| 861 |
if stdin_w is not None: |
| 862 |
stdin.close() |
| 863 |
self._proc.stdin = open(stdin_w.detach(), 'wb', buffering=bufsize) |
| 864 |
stdin_w = None |
| 865 |
finally: |
| 866 |
if stdin_w is not None: |
| 867 |
stdin.close() |
| 868 |
stdin_w.close() |
| 869 |
|
| 870 |
|
| 871 |
class _PidfdChildWatcher: |
| 872 |
"""Child watcher implementation using Linux's pid file descriptors. |
| 873 |
|
| 874 |
This child watcher polls process file descriptors (pidfds) to await child |
| 875 |
process termination. In some respects, PidfdChildWatcher is a "Goldilocks" |
| 876 |
child watcher implementation. It doesn't require signals or threads, doesn't |
| 877 |
interfere with any processes launched outside the event loop, and scales |
| 878 |
linearly with the number of subprocesses launched by the event loop. The |
| 879 |
main disadvantage is that pidfds are specific to Linux, and only work on |
| 880 |
recent (5.3+) kernels. |
| 881 |
""" |
| 882 |
|
| 883 |
def add_child_handler(self, pid, callback, *args): |
| 884 |
loop = events.get_running_loop() |
| 885 |
pidfd = os.pidfd_open(pid) |
| 886 |
loop._add_reader(pidfd, self._do_wait, pid, pidfd, callback, args) |
| 887 |
|
| 888 |
def _do_wait(self, pid, pidfd, callback, args): |
| 889 |
loop = events.get_running_loop() |
| 890 |
loop._remove_reader(pidfd) |
| 891 |
try: |
| 892 |
_, status = os.waitpid(pid, 0) |
| 893 |
except ChildProcessError: |
| 894 |
# The child process is already reaped |
| 895 |
# (may happen if waitpid() is called elsewhere). |
| 896 |
returncode = 255 |
| 897 |
logger.warning( |
| 898 |
"child process pid %d exit status already read: " |
| 899 |
" will report returncode 255", |
| 900 |
pid) |
| 901 |
else: |
| 902 |
returncode = waitstatus_to_exitcode(status) |
| 903 |
|
| 904 |
os.close(pidfd) |
| 905 |
callback(pid, returncode, *args) |
| 906 |
|
| 907 |
class _ThreadedChildWatcher: |
| 908 |
"""Threaded child watcher implementation. |
| 909 |
|
| 910 |
The watcher uses a thread per process |
| 911 |
for waiting for the process finish. |
| 912 |
|
| 913 |
It doesn't require subscription on POSIX signal |
| 914 |
but a thread creation is not free. |
| 915 |
|
| 916 |
The watcher has O(1) complexity, its performance doesn't depend |
| 917 |
on amount of spawn processes. |
| 918 |
""" |
| 919 |
|
| 920 |
def __init__(self): |
| 921 |
self._pid_counter = itertools.count(0) |
| 922 |
self._threads = {} |
| 923 |
|
| 924 |
def __del__(self, _warn=warnings.warn): |
| 925 |
threads = [thread for thread in list(self._threads.values()) |
| 926 |
if thread.is_alive()] |
| 927 |
if threads: |
| 928 |
_warn(f"{self.__class__} has registered but not finished child processes", |
| 929 |
ResourceWarning, |
| 930 |
source=self) |
| 931 |
|
| 932 |
def add_child_handler(self, pid, callback, *args): |
| 933 |
loop = events.get_running_loop() |
| 934 |
thread = threading.Thread(target=self._do_waitpid, |
| 935 |
name=f"asyncio-waitpid-{next(self._pid_counter)}", |
| 936 |
args=(loop, pid, callback, args), |
| 937 |
daemon=True) |
| 938 |
self._threads[pid] = thread |
| 939 |
thread.start() |
| 940 |
|
| 941 |
def _do_waitpid(self, loop, expected_pid, callback, args): |
| 942 |
assert expected_pid > 0 |
| 943 |
|
| 944 |
if hasattr(os, 'waitid'): |
| 945 |
# Wait for the child process using waitid() on platforms which support it. |
| 946 |
# WNOWAIT is used to avoid reaping the child process, allowing the event loop to |
| 947 |
# reap the child process with waitpid() later in event loop thread. |
| 948 |
# This makes the reaping of the child and notification of the return code |
| 949 |
# atomic with respect to the event loop thread. |
| 950 |
try: |
| 951 |
os.waitid(os.P_PID, expected_pid, os.WEXITED | os.WNOWAIT) |
| 952 |
except ChildProcessError: |
| 953 |
# The child process is already reaped |
| 954 |
pass |
| 955 |
if loop.is_closed(): |
| 956 |
# loop is already closed, reap the zombie here so that it is not leaked. |
| 957 |
pid, _ = self._reap(loop, expected_pid) |
| 958 |
logger.warning("Loop %r that handles pid %r is closed", |
| 959 |
loop, pid) |
| 960 |
else: |
| 961 |
try: |
| 962 |
loop.call_soon_threadsafe( |
| 963 |
self._reap_and_notify, loop, expected_pid, |
| 964 |
callback, args) |
| 965 |
except RuntimeError: |
| 966 |
# The event loop was closed concurrently. |
| 967 |
pid, _ = self._reap(loop, expected_pid) |
| 968 |
logger.warning("Loop %r that handles pid %r is closed", |
| 969 |
loop, pid) |
| 970 |
else: |
| 971 |
# Fallback for platforms that don't support waitid(): we have to |
| 972 |
# reap the child here, which is racy with respect to send_signal() |
| 973 |
pid, returncode = self._reap(loop, expected_pid) |
| 974 |
if loop.is_closed(): |
| 975 |
logger.warning("Loop %r that handles pid %r is closed", |
| 976 |
loop, pid) |
| 977 |
else: |
| 978 |
loop.call_soon_threadsafe(callback, pid, returncode, *args) |
| 979 |
|
| 980 |
self._threads.pop(expected_pid) |
| 981 |
|
| 982 |
def _reap_and_notify(self, loop, expected_pid, callback, args): |
| 983 |
pid, returncode = self._reap(loop, expected_pid) |
| 984 |
callback(pid, returncode, *args) |
| 985 |
|
| 986 |
def _reap(self, loop, expected_pid): |
| 987 |
try: |
| 988 |
pid, status = os.waitpid(expected_pid, 0) |
| 989 |
except ChildProcessError: |
| 990 |
# The child process is already reaped |
| 991 |
# (may happen if waitpid() is called elsewhere). |
| 992 |
pid = expected_pid |
| 993 |
returncode = 255 |
| 994 |
logger.warning( |
| 995 |
"Unknown child process pid %d, will report returncode 255", |
| 996 |
pid) |
| 997 |
else: |
| 998 |
returncode = waitstatus_to_exitcode(status) |
| 999 |
if loop.get_debug(): |
| 1000 |
logger.debug('process %s exited with returncode %s', |
| 1001 |
expected_pid, returncode) |
| 1002 |
return pid, returncode |
| 1003 |
|
| 1004 |
def can_use_pidfd(): |
| 1005 |
if not hasattr(os, 'pidfd_open'): |
| 1006 |
return False |
| 1007 |
try: |
| 1008 |
pid = os.getpid() |
| 1009 |
os.close(os.pidfd_open(pid, 0)) |
| 1010 |
except OSError: |
| 1011 |
# blocked by security policy like SECCOMP |
| 1012 |
return False |
| 1013 |
return True |
| 1014 |
|
| 1015 |
|
| 1016 |
class _UnixDefaultEventLoopPolicy(events._BaseDefaultEventLoopPolicy): |
| 1017 |
"""UNIX event loop policy""" |
| 1018 |
_loop_factory = _UnixSelectorEventLoop |
| 1019 |
|
| 1020 |
|
| 1021 |
SelectorEventLoop = _UnixSelectorEventLoop |
| 1022 |
_DefaultEventLoopPolicy = _UnixDefaultEventLoopPolicy |
| 1023 |
EventLoop = SelectorEventLoop |