| 1 |
# |
| 2 |
# A higher level module for using sockets (or Windows named pipes) |
| 3 |
# |
| 4 |
# multiprocessing/connection.py |
| 5 |
# |
| 6 |
# Copyright (c) 2006-2008, R Oudkerk |
| 7 |
# Licensed to PSF under a Contributor Agreement. |
| 8 |
# |
| 9 |
|
| 10 |
__all__ = [ 'Client', 'Listener', 'Pipe', 'wait' ] |
| 11 |
|
| 12 |
import errno |
| 13 |
import io |
| 14 |
import itertools |
| 15 |
import os |
| 16 |
import sys |
| 17 |
import socket |
| 18 |
import struct |
| 19 |
import time |
| 20 |
|
| 21 |
|
| 22 |
from . import util |
| 23 |
|
| 24 |
from . import AuthenticationError, BufferTooShort |
| 25 |
from .context import reduction |
| 26 |
_ForkingPickler = reduction.ForkingPickler |
| 27 |
|
| 28 |
try: |
| 29 |
import _multiprocessing |
| 30 |
import _winapi |
| 31 |
from _winapi import WAIT_OBJECT_0, WAIT_ABANDONED_0, WAIT_TIMEOUT, INFINITE |
| 32 |
except ImportError: |
| 33 |
if sys.platform == 'win32': |
| 34 |
raise |
| 35 |
_winapi = None |
| 36 |
|
| 37 |
# |
| 38 |
# |
| 39 |
# |
| 40 |
|
| 41 |
# 64 KiB is the default PIPE buffer size of most POSIX platforms. |
| 42 |
BUFSIZE = 64 * 1024 |
| 43 |
|
| 44 |
# A very generous timeout when it comes to local connections... |
| 45 |
CONNECTION_TIMEOUT = 20. |
| 46 |
|
| 47 |
_mmap_counter = itertools.count() |
| 48 |
_MAX_PIPE_ATTEMPTS = 100 |
| 49 |
|
| 50 |
default_family = 'AF_INET' |
| 51 |
families = ['AF_INET'] |
| 52 |
|
| 53 |
if hasattr(socket, 'AF_UNIX'): |
| 54 |
default_family = 'AF_UNIX' |
| 55 |
families += ['AF_UNIX'] |
| 56 |
|
| 57 |
if sys.platform == 'win32': |
| 58 |
default_family = 'AF_PIPE' |
| 59 |
families += ['AF_PIPE'] |
| 60 |
|
| 61 |
|
| 62 |
def _init_timeout(timeout=CONNECTION_TIMEOUT): |
| 63 |
return time.monotonic() + timeout |
| 64 |
|
| 65 |
def _check_timeout(t): |
| 66 |
return time.monotonic() > t |
| 67 |
|
| 68 |
# |
| 69 |
# |
| 70 |
# |
| 71 |
|
| 72 |
def arbitrary_address(family): |
| 73 |
''' |
| 74 |
Return an arbitrary free address for the given family |
| 75 |
''' |
| 76 |
if family == 'AF_INET': |
| 77 |
return ('localhost', 0) |
| 78 |
elif family == 'AF_UNIX': |
| 79 |
# NOTE: util.get_temp_dir() is a 0o700 per-process directory. A |
| 80 |
# mktemp-style ToC vs ToU concern is not important; bind() surfaces |
| 81 |
# the extremely unlikely collision as EADDRINUSE. |
| 82 |
return os.path.join(util.get_temp_dir(), |
| 83 |
f'sock-{os.urandom(6).hex()}') |
| 84 |
elif family == 'AF_PIPE': |
| 85 |
return (r'\\.\pipe\pyc-%d-%d-%s' % |
| 86 |
(os.getpid(), next(_mmap_counter), os.urandom(8).hex())) |
| 87 |
else: |
| 88 |
raise ValueError('unrecognized family') |
| 89 |
|
| 90 |
def _validate_family(family): |
| 91 |
''' |
| 92 |
Checks if the family is valid for the current environment. |
| 93 |
''' |
| 94 |
if sys.platform != 'win32' and family == 'AF_PIPE': |
| 95 |
raise ValueError('Family %s is not recognized.' % family) |
| 96 |
|
| 97 |
if sys.platform == 'win32' and family == 'AF_UNIX': |
| 98 |
# double check |
| 99 |
if not hasattr(socket, family): |
| 100 |
raise ValueError('Family %s is not recognized.' % family) |
| 101 |
|
| 102 |
def address_type(address): |
| 103 |
''' |
| 104 |
Return the types of the address |
| 105 |
|
| 106 |
This can be 'AF_INET', 'AF_UNIX', or 'AF_PIPE' |
| 107 |
''' |
| 108 |
if type(address) == tuple: |
| 109 |
return 'AF_INET' |
| 110 |
elif type(address) is str and address.startswith('\\\\'): |
| 111 |
return 'AF_PIPE' |
| 112 |
elif type(address) is str or util.is_abstract_socket_namespace(address): |
| 113 |
return 'AF_UNIX' |
| 114 |
else: |
| 115 |
raise ValueError('address type of %r unrecognized' % address) |
| 116 |
|
| 117 |
# |
| 118 |
# Connection classes |
| 119 |
# |
| 120 |
|
| 121 |
class _ConnectionBase: |
| 122 |
_handle = None |
| 123 |
|
| 124 |
def __init__(self, handle, readable=True, writable=True): |
| 125 |
handle = handle.__index__() |
| 126 |
if handle < 0: |
| 127 |
raise ValueError("invalid handle") |
| 128 |
if not readable and not writable: |
| 129 |
raise ValueError( |
| 130 |
"at least one of `readable` and `writable` must be True") |
| 131 |
self._handle = handle |
| 132 |
self._readable = readable |
| 133 |
self._writable = writable |
| 134 |
|
| 135 |
# XXX should we use util.Finalize instead of a __del__? |
| 136 |
|
| 137 |
def __del__(self): |
| 138 |
if self._handle is not None: |
| 139 |
self._close() |
| 140 |
|
| 141 |
def _check_closed(self): |
| 142 |
if self._handle is None: |
| 143 |
raise OSError("handle is closed") |
| 144 |
|
| 145 |
def _check_readable(self): |
| 146 |
if not self._readable: |
| 147 |
raise OSError("connection is write-only") |
| 148 |
|
| 149 |
def _check_writable(self): |
| 150 |
if not self._writable: |
| 151 |
raise OSError("connection is read-only") |
| 152 |
|
| 153 |
def _bad_message_length(self): |
| 154 |
if self._writable: |
| 155 |
self._readable = False |
| 156 |
else: |
| 157 |
self.close() |
| 158 |
raise OSError("bad message length") |
| 159 |
|
| 160 |
@property |
| 161 |
def closed(self): |
| 162 |
"""True if the connection is closed""" |
| 163 |
return self._handle is None |
| 164 |
|
| 165 |
@property |
| 166 |
def readable(self): |
| 167 |
"""True if the connection is readable""" |
| 168 |
return self._readable |
| 169 |
|
| 170 |
@property |
| 171 |
def writable(self): |
| 172 |
"""True if the connection is writable""" |
| 173 |
return self._writable |
| 174 |
|
| 175 |
def fileno(self): |
| 176 |
"""File descriptor or handle of the connection""" |
| 177 |
self._check_closed() |
| 178 |
return self._handle |
| 179 |
|
| 180 |
def close(self): |
| 181 |
"""Close the connection""" |
| 182 |
if self._handle is not None: |
| 183 |
try: |
| 184 |
self._close() |
| 185 |
finally: |
| 186 |
self._handle = None |
| 187 |
|
| 188 |
def _detach(self): |
| 189 |
"""Stop managing the underlying file descriptor or handle.""" |
| 190 |
self._handle = None |
| 191 |
|
| 192 |
def send_bytes(self, buf, offset=0, size=None): |
| 193 |
"""Send the bytes data from a bytes-like object""" |
| 194 |
self._check_closed() |
| 195 |
self._check_writable() |
| 196 |
m = memoryview(buf) |
| 197 |
if m.itemsize > 1: |
| 198 |
m = m.cast('B') |
| 199 |
n = m.nbytes |
| 200 |
if offset < 0: |
| 201 |
raise ValueError("offset is negative") |
| 202 |
if n < offset: |
| 203 |
raise ValueError("buffer length < offset") |
| 204 |
if size is None: |
| 205 |
size = n - offset |
| 206 |
elif size < 0: |
| 207 |
raise ValueError("size is negative") |
| 208 |
elif offset + size > n: |
| 209 |
raise ValueError("buffer length < offset + size") |
| 210 |
self._send_bytes(m[offset:offset + size]) |
| 211 |
|
| 212 |
def send(self, obj): |
| 213 |
"""Send a (picklable) object""" |
| 214 |
self._check_closed() |
| 215 |
self._check_writable() |
| 216 |
self._send_bytes(_ForkingPickler.dumps(obj)) |
| 217 |
|
| 218 |
def recv_bytes(self, maxlength=None): |
| 219 |
""" |
| 220 |
Receive bytes data as a bytes object. |
| 221 |
""" |
| 222 |
self._check_closed() |
| 223 |
self._check_readable() |
| 224 |
if maxlength is not None and maxlength < 0: |
| 225 |
raise ValueError("negative maxlength") |
| 226 |
buf = self._recv_bytes(maxlength) |
| 227 |
if buf is None: |
| 228 |
self._bad_message_length() |
| 229 |
return buf.getvalue() |
| 230 |
|
| 231 |
def recv_bytes_into(self, buf, offset=0): |
| 232 |
""" |
| 233 |
Receive bytes data into a writeable bytes-like object. |
| 234 |
Return the number of bytes read. |
| 235 |
""" |
| 236 |
self._check_closed() |
| 237 |
self._check_readable() |
| 238 |
with memoryview(buf) as m: |
| 239 |
# Get bytesize of arbitrary buffer |
| 240 |
itemsize = m.itemsize |
| 241 |
bytesize = itemsize * len(m) |
| 242 |
if offset < 0: |
| 243 |
raise ValueError("negative offset") |
| 244 |
elif offset > bytesize: |
| 245 |
raise ValueError("offset too large") |
| 246 |
result = self._recv_bytes() |
| 247 |
size = result.tell() |
| 248 |
if bytesize < offset + size: |
| 249 |
raise BufferTooShort(result.getvalue()) |
| 250 |
# Message can fit in dest |
| 251 |
result.seek(0) |
| 252 |
result.readinto(m[offset // itemsize : |
| 253 |
(offset + size) // itemsize]) |
| 254 |
return size |
| 255 |
|
| 256 |
def recv(self): |
| 257 |
"""Receive a (picklable) object""" |
| 258 |
self._check_closed() |
| 259 |
self._check_readable() |
| 260 |
buf = self._recv_bytes() |
| 261 |
return _ForkingPickler.loads(buf.getbuffer()) |
| 262 |
|
| 263 |
def poll(self, timeout=0.0): |
| 264 |
"""Whether there is any input available to be read""" |
| 265 |
self._check_closed() |
| 266 |
self._check_readable() |
| 267 |
return self._poll(timeout) |
| 268 |
|
| 269 |
def __enter__(self): |
| 270 |
return self |
| 271 |
|
| 272 |
def __exit__(self, exc_type, exc_value, exc_tb): |
| 273 |
self.close() |
| 274 |
|
| 275 |
|
| 276 |
if _winapi: |
| 277 |
|
| 278 |
class PipeConnection(_ConnectionBase): |
| 279 |
""" |
| 280 |
Connection class based on a Windows named pipe. |
| 281 |
Overlapped I/O is used, so the handles must have been created |
| 282 |
with FILE_FLAG_OVERLAPPED. |
| 283 |
""" |
| 284 |
_got_empty_message = False |
| 285 |
_send_ov = None |
| 286 |
|
| 287 |
def _close(self, _CloseHandle=_winapi.CloseHandle): |
| 288 |
ov = self._send_ov |
| 289 |
if ov is not None: |
| 290 |
# Interrupt WaitForMultipleObjects() in _send_bytes() |
| 291 |
ov.cancel() |
| 292 |
_CloseHandle(self._handle) |
| 293 |
|
| 294 |
def _send_bytes(self, buf): |
| 295 |
if self._send_ov is not None: |
| 296 |
# A connection should only be used by a single thread |
| 297 |
raise ValueError("concurrent send_bytes() calls " |
| 298 |
"are not supported") |
| 299 |
ov, err = _winapi.WriteFile(self._handle, buf, overlapped=True) |
| 300 |
self._send_ov = ov |
| 301 |
try: |
| 302 |
if err == _winapi.ERROR_IO_PENDING: |
| 303 |
waitres = _winapi.WaitForMultipleObjects( |
| 304 |
[ov.event], False, INFINITE) |
| 305 |
assert waitres == WAIT_OBJECT_0 |
| 306 |
except: |
| 307 |
ov.cancel() |
| 308 |
raise |
| 309 |
finally: |
| 310 |
self._send_ov = None |
| 311 |
nwritten, err = ov.GetOverlappedResult(True) |
| 312 |
if err == _winapi.ERROR_OPERATION_ABORTED: |
| 313 |
# close() was called by another thread while |
| 314 |
# WaitForMultipleObjects() was waiting for the overlapped |
| 315 |
# operation. |
| 316 |
raise OSError(errno.EPIPE, "handle is closed") |
| 317 |
assert err == 0 |
| 318 |
assert nwritten == len(buf) |
| 319 |
|
| 320 |
def _recv_bytes(self, maxsize=None): |
| 321 |
if self._got_empty_message: |
| 322 |
self._got_empty_message = False |
| 323 |
return io.BytesIO() |
| 324 |
else: |
| 325 |
bsize = 128 if maxsize is None else min(maxsize, 128) |
| 326 |
try: |
| 327 |
ov, err = _winapi.ReadFile(self._handle, bsize, |
| 328 |
overlapped=True) |
| 329 |
|
| 330 |
sentinel = object() |
| 331 |
return_value = sentinel |
| 332 |
try: |
| 333 |
try: |
| 334 |
if err == _winapi.ERROR_IO_PENDING: |
| 335 |
waitres = _winapi.WaitForMultipleObjects( |
| 336 |
[ov.event], False, INFINITE) |
| 337 |
assert waitres == WAIT_OBJECT_0 |
| 338 |
except: |
| 339 |
ov.cancel() |
| 340 |
raise |
| 341 |
finally: |
| 342 |
nread, err = ov.GetOverlappedResult(True) |
| 343 |
if err == 0: |
| 344 |
f = io.BytesIO() |
| 345 |
f.write(ov.getbuffer()) |
| 346 |
return_value = f |
| 347 |
elif err == _winapi.ERROR_MORE_DATA: |
| 348 |
return_value = self._get_more_data(ov, maxsize) |
| 349 |
except: |
| 350 |
if return_value is sentinel: |
| 351 |
raise |
| 352 |
|
| 353 |
if return_value is not sentinel: |
| 354 |
return return_value |
| 355 |
except OSError as e: |
| 356 |
if e.winerror == _winapi.ERROR_BROKEN_PIPE: |
| 357 |
raise EOFError |
| 358 |
else: |
| 359 |
raise |
| 360 |
raise RuntimeError("shouldn't get here; expected KeyboardInterrupt") |
| 361 |
|
| 362 |
def _poll(self, timeout): |
| 363 |
if (self._got_empty_message or |
| 364 |
_winapi.PeekNamedPipe(self._handle)[0] != 0): |
| 365 |
return True |
| 366 |
return bool(wait([self], timeout)) |
| 367 |
|
| 368 |
def _get_more_data(self, ov, maxsize): |
| 369 |
buf = ov.getbuffer() |
| 370 |
f = io.BytesIO() |
| 371 |
f.write(buf) |
| 372 |
left = _winapi.PeekNamedPipe(self._handle)[1] |
| 373 |
assert left > 0 |
| 374 |
if maxsize is not None and len(buf) + left > maxsize: |
| 375 |
self._bad_message_length() |
| 376 |
ov, err = _winapi.ReadFile(self._handle, left, overlapped=True) |
| 377 |
rbytes, err = ov.GetOverlappedResult(True) |
| 378 |
assert err == 0 |
| 379 |
assert rbytes == left |
| 380 |
f.write(ov.getbuffer()) |
| 381 |
return f |
| 382 |
|
| 383 |
|
| 384 |
class Connection(_ConnectionBase): |
| 385 |
""" |
| 386 |
Connection class based on an arbitrary file descriptor (Unix only), or |
| 387 |
a socket handle (Windows). |
| 388 |
""" |
| 389 |
|
| 390 |
if _winapi: |
| 391 |
def _close(self, _close=_multiprocessing.closesocket): |
| 392 |
_close(self._handle) |
| 393 |
_write = _multiprocessing.send |
| 394 |
_read = _multiprocessing.recv |
| 395 |
else: |
| 396 |
def _close(self, _close=os.close): |
| 397 |
_close(self._handle) |
| 398 |
_write = os.write |
| 399 |
_read = os.read |
| 400 |
|
| 401 |
def _send(self, buf, write=_write): |
| 402 |
remaining = len(buf) |
| 403 |
while True: |
| 404 |
n = write(self._handle, buf) |
| 405 |
remaining -= n |
| 406 |
if remaining == 0: |
| 407 |
break |
| 408 |
buf = buf[n:] |
| 409 |
|
| 410 |
def _recv(self, size, read=_read): |
| 411 |
buf = io.BytesIO() |
| 412 |
handle = self._handle |
| 413 |
remaining = size |
| 414 |
while remaining > 0: |
| 415 |
to_read = min(BUFSIZE, remaining) |
| 416 |
chunk = read(handle, to_read) |
| 417 |
n = len(chunk) |
| 418 |
if n == 0: |
| 419 |
if remaining == size: |
| 420 |
raise EOFError |
| 421 |
else: |
| 422 |
raise OSError("got end of file during message") |
| 423 |
buf.write(chunk) |
| 424 |
remaining -= n |
| 425 |
return buf |
| 426 |
|
| 427 |
def _send_bytes(self, buf): |
| 428 |
n = len(buf) |
| 429 |
if n > 0x7fffffff: |
| 430 |
pre_header = struct.pack("!i", -1) |
| 431 |
header = struct.pack("!Q", n) |
| 432 |
self._send(pre_header) |
| 433 |
self._send(header) |
| 434 |
self._send(buf) |
| 435 |
else: |
| 436 |
# For wire compatibility with 3.7 and lower |
| 437 |
header = struct.pack("!i", n) |
| 438 |
if n > 16384: |
| 439 |
# The payload is large so Nagle's algorithm won't be triggered |
| 440 |
# and we'd better avoid the cost of concatenation. |
| 441 |
self._send(header) |
| 442 |
self._send(buf) |
| 443 |
else: |
| 444 |
# Issue #20540: concatenate before sending, to avoid delays due |
| 445 |
# to Nagle's algorithm on a TCP socket. |
| 446 |
# Also note we want to avoid sending a 0-length buffer separately, |
| 447 |
# to avoid "broken pipe" errors if the other end closed the pipe. |
| 448 |
self._send(header + buf) |
| 449 |
|
| 450 |
def _recv_bytes(self, maxsize=None): |
| 451 |
buf = self._recv(4) |
| 452 |
size, = struct.unpack("!i", buf.getvalue()) |
| 453 |
if size == -1: |
| 454 |
buf = self._recv(8) |
| 455 |
size, = struct.unpack("!Q", buf.getvalue()) |
| 456 |
if maxsize is not None and size > maxsize: |
| 457 |
return None |
| 458 |
return self._recv(size) |
| 459 |
|
| 460 |
def _poll(self, timeout): |
| 461 |
r = wait([self], timeout) |
| 462 |
return bool(r) |
| 463 |
|
| 464 |
|
| 465 |
# |
| 466 |
# Public functions |
| 467 |
# |
| 468 |
|
| 469 |
class Listener(object): |
| 470 |
''' |
| 471 |
Returns a listener object. |
| 472 |
|
| 473 |
This is a wrapper for a bound socket which is 'listening' for |
| 474 |
connections, or for a Windows named pipe. |
| 475 |
''' |
| 476 |
def __init__(self, address=None, family=None, backlog=1, authkey=None): |
| 477 |
family = family or (address and address_type(address)) \ |
| 478 |
or default_family |
| 479 |
_validate_family(family) |
| 480 |
if authkey is not None and not isinstance(authkey, bytes): |
| 481 |
raise TypeError('authkey should be a byte string') |
| 482 |
|
| 483 |
if family == 'AF_PIPE': |
| 484 |
if address: |
| 485 |
self._listener = PipeListener(address, backlog) |
| 486 |
else: |
| 487 |
for attempts in itertools.count(): |
| 488 |
address = arbitrary_address(family) |
| 489 |
try: |
| 490 |
self._listener = PipeListener(address, backlog) |
| 491 |
break |
| 492 |
except OSError as e: |
| 493 |
if attempts >= _MAX_PIPE_ATTEMPTS: |
| 494 |
raise |
| 495 |
if e.winerror not in (_winapi.ERROR_PIPE_BUSY, |
| 496 |
_winapi.ERROR_ACCESS_DENIED): |
| 497 |
raise |
| 498 |
else: |
| 499 |
address = address or arbitrary_address(family) |
| 500 |
self._listener = SocketListener(address, family, backlog) |
| 501 |
|
| 502 |
self._authkey = authkey |
| 503 |
|
| 504 |
def accept(self): |
| 505 |
''' |
| 506 |
Accept a connection on the bound socket or named pipe of `self`. |
| 507 |
|
| 508 |
Returns a `Connection` object. |
| 509 |
''' |
| 510 |
if self._listener is None: |
| 511 |
raise OSError('listener is closed') |
| 512 |
|
| 513 |
c = self._listener.accept() |
| 514 |
if self._authkey is not None: |
| 515 |
deliver_challenge(c, self._authkey) |
| 516 |
answer_challenge(c, self._authkey) |
| 517 |
return c |
| 518 |
|
| 519 |
def close(self): |
| 520 |
''' |
| 521 |
Close the bound socket or named pipe of `self`. |
| 522 |
''' |
| 523 |
listener = self._listener |
| 524 |
if listener is not None: |
| 525 |
self._listener = None |
| 526 |
listener.close() |
| 527 |
|
| 528 |
@property |
| 529 |
def address(self): |
| 530 |
return self._listener._address |
| 531 |
|
| 532 |
@property |
| 533 |
def last_accepted(self): |
| 534 |
return self._listener._last_accepted |
| 535 |
|
| 536 |
def __enter__(self): |
| 537 |
return self |
| 538 |
|
| 539 |
def __exit__(self, exc_type, exc_value, exc_tb): |
| 540 |
self.close() |
| 541 |
|
| 542 |
|
| 543 |
def Client(address, family=None, authkey=None): |
| 544 |
''' |
| 545 |
Returns a connection to the address of a `Listener` |
| 546 |
''' |
| 547 |
family = family or address_type(address) |
| 548 |
_validate_family(family) |
| 549 |
if family == 'AF_PIPE': |
| 550 |
c = PipeClient(address) |
| 551 |
else: |
| 552 |
c = SocketClient(address) |
| 553 |
|
| 554 |
if authkey is not None and not isinstance(authkey, bytes): |
| 555 |
raise TypeError('authkey should be a byte string') |
| 556 |
|
| 557 |
if authkey is not None: |
| 558 |
answer_challenge(c, authkey) |
| 559 |
deliver_challenge(c, authkey) |
| 560 |
|
| 561 |
return c |
| 562 |
|
| 563 |
|
| 564 |
if sys.platform != 'win32': |
| 565 |
|
| 566 |
def Pipe(duplex=True): |
| 567 |
''' |
| 568 |
Returns pair of connection objects at either end of a pipe |
| 569 |
''' |
| 570 |
if duplex: |
| 571 |
s1, s2 = socket.socketpair() |
| 572 |
s1.setblocking(True) |
| 573 |
s2.setblocking(True) |
| 574 |
c1 = Connection(s1.detach()) |
| 575 |
c2 = Connection(s2.detach()) |
| 576 |
else: |
| 577 |
fd1, fd2 = os.pipe() |
| 578 |
c1 = Connection(fd1, writable=False) |
| 579 |
c2 = Connection(fd2, readable=False) |
| 580 |
|
| 581 |
return c1, c2 |
| 582 |
|
| 583 |
else: |
| 584 |
|
| 585 |
def Pipe(duplex=True): |
| 586 |
''' |
| 587 |
Returns pair of connection objects at either end of a pipe |
| 588 |
''' |
| 589 |
if duplex: |
| 590 |
openmode = _winapi.PIPE_ACCESS_DUPLEX |
| 591 |
access = _winapi.GENERIC_READ | _winapi.GENERIC_WRITE |
| 592 |
obsize, ibsize = BUFSIZE, BUFSIZE |
| 593 |
else: |
| 594 |
openmode = _winapi.PIPE_ACCESS_INBOUND |
| 595 |
access = _winapi.GENERIC_WRITE |
| 596 |
obsize, ibsize = 0, BUFSIZE |
| 597 |
|
| 598 |
for attempts in itertools.count(): |
| 599 |
address = arbitrary_address('AF_PIPE') |
| 600 |
try: |
| 601 |
h1 = _winapi.CreateNamedPipe( |
| 602 |
address, openmode | _winapi.FILE_FLAG_OVERLAPPED | |
| 603 |
_winapi.FILE_FLAG_FIRST_PIPE_INSTANCE, |
| 604 |
_winapi.PIPE_TYPE_MESSAGE | _winapi.PIPE_READMODE_MESSAGE | |
| 605 |
_winapi.PIPE_WAIT, |
| 606 |
1, obsize, ibsize, _winapi.NMPWAIT_WAIT_FOREVER, |
| 607 |
# default security descriptor: the handle cannot be inherited |
| 608 |
_winapi.NULL |
| 609 |
) |
| 610 |
break |
| 611 |
except OSError as e: |
| 612 |
if attempts >= _MAX_PIPE_ATTEMPTS: |
| 613 |
raise |
| 614 |
if e.winerror not in (_winapi.ERROR_PIPE_BUSY, |
| 615 |
_winapi.ERROR_ACCESS_DENIED): |
| 616 |
raise |
| 617 |
h2 = _winapi.CreateFile( |
| 618 |
address, access, 0, _winapi.NULL, _winapi.OPEN_EXISTING, |
| 619 |
_winapi.FILE_FLAG_OVERLAPPED, _winapi.NULL |
| 620 |
) |
| 621 |
_winapi.SetNamedPipeHandleState( |
| 622 |
h2, _winapi.PIPE_READMODE_MESSAGE, None, None |
| 623 |
) |
| 624 |
|
| 625 |
overlapped = _winapi.ConnectNamedPipe(h1, overlapped=True) |
| 626 |
_, err = overlapped.GetOverlappedResult(True) |
| 627 |
assert err == 0 |
| 628 |
|
| 629 |
c1 = PipeConnection(h1, writable=duplex) |
| 630 |
c2 = PipeConnection(h2, readable=duplex) |
| 631 |
|
| 632 |
return c1, c2 |
| 633 |
|
| 634 |
# |
| 635 |
# Definitions for connections based on sockets |
| 636 |
# |
| 637 |
|
| 638 |
class SocketListener(object): |
| 639 |
''' |
| 640 |
Representation of a socket which is bound to an address and listening |
| 641 |
''' |
| 642 |
def __init__(self, address, family, backlog=1): |
| 643 |
self._socket = socket.socket(getattr(socket, family)) |
| 644 |
try: |
| 645 |
# SO_REUSEADDR has different semantics on Windows (issue #2550). |
| 646 |
if os.name == 'posix': |
| 647 |
self._socket.setsockopt(socket.SOL_SOCKET, |
| 648 |
socket.SO_REUSEADDR, 1) |
| 649 |
self._socket.setblocking(True) |
| 650 |
self._socket.bind(address) |
| 651 |
self._socket.listen(backlog) |
| 652 |
self._address = self._socket.getsockname() |
| 653 |
except OSError: |
| 654 |
self._socket.close() |
| 655 |
raise |
| 656 |
self._family = family |
| 657 |
self._last_accepted = None |
| 658 |
|
| 659 |
if family == 'AF_UNIX' and not util.is_abstract_socket_namespace(address): |
| 660 |
# Linux abstract socket namespaces do not need to be explicitly unlinked |
| 661 |
self._unlink = util.Finalize( |
| 662 |
self, os.unlink, args=(address,), exitpriority=0 |
| 663 |
) |
| 664 |
else: |
| 665 |
self._unlink = None |
| 666 |
|
| 667 |
def accept(self): |
| 668 |
s, self._last_accepted = self._socket.accept() |
| 669 |
s.setblocking(True) |
| 670 |
return Connection(s.detach()) |
| 671 |
|
| 672 |
def close(self): |
| 673 |
try: |
| 674 |
self._socket.close() |
| 675 |
finally: |
| 676 |
unlink = self._unlink |
| 677 |
if unlink is not None: |
| 678 |
self._unlink = None |
| 679 |
unlink() |
| 680 |
|
| 681 |
|
| 682 |
def SocketClient(address): |
| 683 |
''' |
| 684 |
Return a connection object connected to the socket given by `address` |
| 685 |
''' |
| 686 |
family = address_type(address) |
| 687 |
with socket.socket( getattr(socket, family) ) as s: |
| 688 |
s.setblocking(True) |
| 689 |
s.connect(address) |
| 690 |
return Connection(s.detach()) |
| 691 |
|
| 692 |
# |
| 693 |
# Definitions for connections based on named pipes |
| 694 |
# |
| 695 |
|
| 696 |
if sys.platform == 'win32': |
| 697 |
|
| 698 |
class PipeListener(object): |
| 699 |
''' |
| 700 |
Representation of a named pipe |
| 701 |
''' |
| 702 |
def __init__(self, address, backlog=None): |
| 703 |
self._address = address |
| 704 |
self._handle_queue = [self._new_handle(first=True)] |
| 705 |
|
| 706 |
self._last_accepted = None |
| 707 |
util.sub_debug('listener created with address=%r', self._address) |
| 708 |
self.close = util.Finalize( |
| 709 |
self, PipeListener._finalize_pipe_listener, |
| 710 |
args=(self._handle_queue, self._address), exitpriority=0 |
| 711 |
) |
| 712 |
|
| 713 |
def _new_handle(self, first=False): |
| 714 |
flags = _winapi.PIPE_ACCESS_DUPLEX | _winapi.FILE_FLAG_OVERLAPPED |
| 715 |
if first: |
| 716 |
flags |= _winapi.FILE_FLAG_FIRST_PIPE_INSTANCE |
| 717 |
return _winapi.CreateNamedPipe( |
| 718 |
self._address, flags, |
| 719 |
_winapi.PIPE_TYPE_MESSAGE | _winapi.PIPE_READMODE_MESSAGE | |
| 720 |
_winapi.PIPE_WAIT, |
| 721 |
_winapi.PIPE_UNLIMITED_INSTANCES, BUFSIZE, BUFSIZE, |
| 722 |
_winapi.NMPWAIT_WAIT_FOREVER, _winapi.NULL |
| 723 |
) |
| 724 |
|
| 725 |
def accept(self): |
| 726 |
self._handle_queue.append(self._new_handle()) |
| 727 |
handle = self._handle_queue.pop(0) |
| 728 |
try: |
| 729 |
ov = _winapi.ConnectNamedPipe(handle, overlapped=True) |
| 730 |
except OSError as e: |
| 731 |
if e.winerror != _winapi.ERROR_NO_DATA: |
| 732 |
raise |
| 733 |
# ERROR_NO_DATA can occur if a client has already connected, |
| 734 |
# written data and then disconnected -- see Issue 14725. |
| 735 |
else: |
| 736 |
try: |
| 737 |
res = _winapi.WaitForMultipleObjects( |
| 738 |
[ov.event], False, INFINITE) |
| 739 |
except: |
| 740 |
ov.cancel() |
| 741 |
_winapi.CloseHandle(handle) |
| 742 |
raise |
| 743 |
finally: |
| 744 |
_, err = ov.GetOverlappedResult(True) |
| 745 |
assert err == 0 |
| 746 |
return PipeConnection(handle) |
| 747 |
|
| 748 |
@staticmethod |
| 749 |
def _finalize_pipe_listener(queue, address): |
| 750 |
util.sub_debug('closing listener with address=%r', address) |
| 751 |
for handle in queue: |
| 752 |
_winapi.CloseHandle(handle) |
| 753 |
|
| 754 |
def PipeClient(address): |
| 755 |
''' |
| 756 |
Return a connection object connected to the pipe given by `address` |
| 757 |
''' |
| 758 |
t = _init_timeout() |
| 759 |
while 1: |
| 760 |
try: |
| 761 |
_winapi.WaitNamedPipe(address, 1000) |
| 762 |
h = _winapi.CreateFile( |
| 763 |
address, _winapi.GENERIC_READ | _winapi.GENERIC_WRITE, |
| 764 |
0, _winapi.NULL, _winapi.OPEN_EXISTING, |
| 765 |
_winapi.FILE_FLAG_OVERLAPPED, _winapi.NULL |
| 766 |
) |
| 767 |
except OSError as e: |
| 768 |
if e.winerror not in (_winapi.ERROR_SEM_TIMEOUT, |
| 769 |
_winapi.ERROR_PIPE_BUSY) or _check_timeout(t): |
| 770 |
raise |
| 771 |
else: |
| 772 |
break |
| 773 |
else: |
| 774 |
raise |
| 775 |
|
| 776 |
_winapi.SetNamedPipeHandleState( |
| 777 |
h, _winapi.PIPE_READMODE_MESSAGE, None, None |
| 778 |
) |
| 779 |
return PipeConnection(h) |
| 780 |
|
| 781 |
# |
| 782 |
# Authentication stuff |
| 783 |
# |
| 784 |
|
| 785 |
MESSAGE_LENGTH = 40 # MUST be > 20 |
| 786 |
|
| 787 |
_CHALLENGE = b'#CHALLENGE#' |
| 788 |
_WELCOME = b'#WELCOME#' |
| 789 |
_FAILURE = b'#FAILURE#' |
| 790 |
|
| 791 |
# multiprocessing.connection Authentication Handshake Protocol Description |
| 792 |
# (as documented for reference after reading the existing code) |
| 793 |
# ============================================================================= |
| 794 |
# |
| 795 |
# On Windows: native pipes with "overlapped IO" are used to send the bytes, |
| 796 |
# instead of the length prefix SIZE scheme described below. (ie: the OS deals |
| 797 |
# with message sizes for us) |
| 798 |
# |
| 799 |
# Protocol error behaviors: |
| 800 |
# |
| 801 |
# On POSIX, any failure to receive the length prefix into SIZE, for SIZE greater |
| 802 |
# than the requested maxsize to receive, or receiving fewer than SIZE bytes |
| 803 |
# results in the connection being closed and auth to fail. |
| 804 |
# |
| 805 |
# On Windows, receiving too few bytes is never a low level _recv_bytes read |
| 806 |
# error, receiving too many will trigger an error only if receive maxsize |
| 807 |
# value was larger than 128 OR the if the data arrived in smaller pieces. |
| 808 |
# |
| 809 |
# Serving side Client side |
| 810 |
# ------------------------------ --------------------------------------- |
| 811 |
# 0. Open a connection on the pipe. |
| 812 |
# 1. Accept connection. |
| 813 |
# 2. Random 20+ bytes -> MESSAGE |
| 814 |
# Modern servers always send |
| 815 |
# more than 20 bytes and include |
| 816 |
# a {digest} prefix on it with |
| 817 |
# their preferred HMAC digest. |
| 818 |
# Legacy ones send ==20 bytes. |
| 819 |
# 3. send 4 byte length (net order) |
| 820 |
# prefix followed by: |
| 821 |
# b'#CHALLENGE#' + MESSAGE |
| 822 |
# 4. Receive 4 bytes, parse as network byte |
| 823 |
# order integer. If it is -1, receive an |
| 824 |
# additional 8 bytes, parse that as network |
| 825 |
# byte order. The result is the length of |
| 826 |
# the data that follows -> SIZE. |
| 827 |
# 5. Receive min(SIZE, 256) bytes -> M1 |
| 828 |
# 6. Assert that M1 starts with: |
| 829 |
# b'#CHALLENGE#' |
| 830 |
# 7. Strip that prefix from M1 into -> M2 |
| 831 |
# 7.1. Parse M2: if it is exactly 20 bytes in |
| 832 |
# length this indicates a legacy server |
| 833 |
# supporting only HMAC-MD5. Otherwise the |
| 834 |
# 7.2. preferred digest is looked up from an |
| 835 |
# expected "{digest}" prefix on M2. No prefix |
| 836 |
# or unsupported digest? <- AuthenticationError |
| 837 |
# 7.3. Put divined algorithm name in -> D_NAME |
| 838 |
# 8. Compute HMAC-D_NAME of AUTHKEY, M2 -> C_DIGEST |
| 839 |
# 9. Send 4 byte length prefix (net order) |
| 840 |
# followed by C_DIGEST bytes. |
| 841 |
# 10. Receive 4 or 4+8 byte length |
| 842 |
# prefix (#4 dance) -> SIZE. |
| 843 |
# 11. Receive min(SIZE, 256) -> C_D. |
| 844 |
# 11.1. Parse C_D: legacy servers |
| 845 |
# accept it as is, "md5" -> D_NAME |
| 846 |
# 11.2. modern servers check the length |
| 847 |
# of C_D, IF it is 16 bytes? |
| 848 |
# 11.2.1. "md5" -> D_NAME |
| 849 |
# and skip to step 12. |
| 850 |
# 11.3. longer? expect and parse a "{digest}" |
| 851 |
# prefix into -> D_NAME. |
| 852 |
# Strip the prefix and store remaining |
| 853 |
# bytes in -> C_D. |
| 854 |
# 11.4. Don't like D_NAME? <- AuthenticationError |
| 855 |
# 12. Compute HMAC-D_NAME of AUTHKEY, |
| 856 |
# MESSAGE into -> M_DIGEST. |
| 857 |
# 13. Compare M_DIGEST == C_D: |
| 858 |
# 14a: Match? Send length prefix & |
| 859 |
# b'#WELCOME#' |
| 860 |
# <- RETURN |
| 861 |
# 14b: Mismatch? Send len prefix & |
| 862 |
# b'#FAILURE#' |
| 863 |
# <- CLOSE & AuthenticationError |
| 864 |
# 15. Receive 4 or 4+8 byte length prefix (net |
| 865 |
# order) again as in #4 into -> SIZE. |
| 866 |
# 16. Receive min(SIZE, 256) bytes -> M3. |
| 867 |
# 17. Compare M3 == b'#WELCOME#': |
| 868 |
# 17a. Match? <- RETURN |
| 869 |
# 17b. Mismatch? <- CLOSE & AuthenticationError |
| 870 |
# |
| 871 |
# If this RETURNed, the connection remains open: it has been authenticated. |
| 872 |
# |
| 873 |
# Length prefixes are used consistently. Even on the legacy protocol, this |
| 874 |
# was good fortune and allowed us to evolve the protocol by using the length |
| 875 |
# of the opening challenge or length of the returned digest as a signal as |
| 876 |
# to which protocol the other end supports. |
| 877 |
|
| 878 |
_ALLOWED_DIGESTS = frozenset( |
| 879 |
{b'md5', b'sha256', b'sha384', b'sha3_256', b'sha3_384'}) |
| 880 |
_MAX_DIGEST_LEN = max(len(_) for _ in _ALLOWED_DIGESTS) |
| 881 |
|
| 882 |
# Old hmac-md5 only server versions from Python <=3.11 sent a message of this |
| 883 |
# length. It happens to not match the length of any supported digest so we can |
| 884 |
# use a message of this length to indicate that we should work in backwards |
| 885 |
# compatible md5-only mode without a {digest_name} prefix on our response. |
| 886 |
_MD5ONLY_MESSAGE_LENGTH = 20 |
| 887 |
_MD5_DIGEST_LEN = 16 |
| 888 |
_LEGACY_LENGTHS = (_MD5ONLY_MESSAGE_LENGTH, _MD5_DIGEST_LEN) |
| 889 |
|
| 890 |
|
| 891 |
def _get_digest_name_and_payload(message): # type: (bytes) -> tuple[str, bytes] |
| 892 |
"""Returns a digest name and the payload for a response hash. |
| 893 |
|
| 894 |
If a legacy protocol is detected based on the message length |
| 895 |
or contents the digest name returned will be empty to indicate |
| 896 |
legacy mode where MD5 and no digest prefix should be sent. |
| 897 |
""" |
| 898 |
# modern message format: b"{digest}payload" longer than 20 bytes |
| 899 |
# legacy message format: 16 or 20 byte b"payload" |
| 900 |
if len(message) in _LEGACY_LENGTHS: |
| 901 |
# Either this was a legacy server challenge, or we're processing |
| 902 |
# a reply from a legacy client that sent an unprefixed 16-byte |
| 903 |
# HMAC-MD5 response. All messages using the modern protocol will |
| 904 |
# be longer than either of these lengths. |
| 905 |
return '', message |
| 906 |
if (message.startswith(b'{') and |
| 907 |
(curly := message.find(b'}', 1, _MAX_DIGEST_LEN+2)) > 0): |
| 908 |
digest = message[1:curly] |
| 909 |
if digest in _ALLOWED_DIGESTS: |
| 910 |
payload = message[curly+1:] |
| 911 |
return digest.decode('ascii'), payload |
| 912 |
raise AuthenticationError( |
| 913 |
'unsupported message length, missing digest prefix, ' |
| 914 |
f'or unsupported digest: {message=}') |
| 915 |
|
| 916 |
|
| 917 |
def _create_response(authkey, message): |
| 918 |
"""Create a MAC based on authkey and message |
| 919 |
|
| 920 |
The MAC algorithm defaults to HMAC-MD5, unless MD5 is not available or |
| 921 |
the message has a '{digest_name}' prefix. For legacy HMAC-MD5, the response |
| 922 |
is the raw MAC, otherwise the response is prefixed with '{digest_name}', |
| 923 |
e.g. b'{sha256}abcdefg...' |
| 924 |
|
| 925 |
Note: The MAC protects the entire message including the digest_name prefix. |
| 926 |
""" |
| 927 |
import hmac |
| 928 |
digest_name = _get_digest_name_and_payload(message)[0] |
| 929 |
# The MAC protects the entire message: digest header and payload. |
| 930 |
if not digest_name: |
| 931 |
# Legacy server without a {digest} prefix on message. |
| 932 |
# Generate a legacy non-prefixed HMAC-MD5 reply. |
| 933 |
try: |
| 934 |
return hmac.new(authkey, message, 'md5').digest() |
| 935 |
except ValueError: |
| 936 |
# HMAC-MD5 is not available (FIPS mode?), fall back to |
| 937 |
# HMAC-SHA2-256 modern protocol. The legacy server probably |
| 938 |
# doesn't support it and will reject us anyways. :shrug: |
| 939 |
digest_name = 'sha256' |
| 940 |
# Modern protocol, indicate the digest used in the reply. |
| 941 |
response = hmac.new(authkey, message, digest_name).digest() |
| 942 |
return b'{%s}%s' % (digest_name.encode('ascii'), response) |
| 943 |
|
| 944 |
|
| 945 |
def _verify_challenge(authkey, message, response): |
| 946 |
"""Verify MAC challenge |
| 947 |
|
| 948 |
If our message did not include a digest_name prefix, the client is allowed |
| 949 |
to select a stronger digest_name from _ALLOWED_DIGESTS. |
| 950 |
|
| 951 |
In case our message is prefixed, a client cannot downgrade to a weaker |
| 952 |
algorithm, because the MAC is calculated over the entire message |
| 953 |
including the '{digest_name}' prefix. |
| 954 |
""" |
| 955 |
import hmac |
| 956 |
response_digest, response_mac = _get_digest_name_and_payload(response) |
| 957 |
response_digest = response_digest or 'md5' |
| 958 |
try: |
| 959 |
expected = hmac.new(authkey, message, response_digest).digest() |
| 960 |
except ValueError: |
| 961 |
raise AuthenticationError(f'{response_digest=} unsupported') |
| 962 |
if len(expected) != len(response_mac): |
| 963 |
raise AuthenticationError( |
| 964 |
f'expected {response_digest!r} of length {len(expected)} ' |
| 965 |
f'got {len(response_mac)}') |
| 966 |
if not hmac.compare_digest(expected, response_mac): |
| 967 |
raise AuthenticationError('digest received was wrong') |
| 968 |
|
| 969 |
|
| 970 |
def deliver_challenge(connection, authkey: bytes, digest_name='sha256'): |
| 971 |
if not isinstance(authkey, bytes): |
| 972 |
raise ValueError( |
| 973 |
"Authkey must be bytes, not {0!s}".format(type(authkey))) |
| 974 |
assert MESSAGE_LENGTH > _MD5ONLY_MESSAGE_LENGTH, "protocol constraint" |
| 975 |
message = os.urandom(MESSAGE_LENGTH) |
| 976 |
message = b'{%s}%s' % (digest_name.encode('ascii'), message) |
| 977 |
# Even when sending a challenge to a legacy client that does not support |
| 978 |
# digest prefixes, they'll take the entire thing as a challenge and |
| 979 |
# respond to it with a raw HMAC-MD5. |
| 980 |
connection.send_bytes(_CHALLENGE + message) |
| 981 |
response = connection.recv_bytes(256) # reject large message |
| 982 |
try: |
| 983 |
_verify_challenge(authkey, message, response) |
| 984 |
except AuthenticationError: |
| 985 |
connection.send_bytes(_FAILURE) |
| 986 |
raise |
| 987 |
else: |
| 988 |
connection.send_bytes(_WELCOME) |
| 989 |
|
| 990 |
|
| 991 |
def answer_challenge(connection, authkey: bytes): |
| 992 |
if not isinstance(authkey, bytes): |
| 993 |
raise ValueError( |
| 994 |
"Authkey must be bytes, not {0!s}".format(type(authkey))) |
| 995 |
message = connection.recv_bytes(256) # reject large message |
| 996 |
if not message.startswith(_CHALLENGE): |
| 997 |
raise AuthenticationError( |
| 998 |
f'Protocol error, expected challenge: {message=}') |
| 999 |
message = message[len(_CHALLENGE):] |
| 1000 |
if len(message) < _MD5ONLY_MESSAGE_LENGTH: |
| 1001 |
raise AuthenticationError(f'challenge too short: {len(message)} bytes') |
| 1002 |
digest = _create_response(authkey, message) |
| 1003 |
connection.send_bytes(digest) |
| 1004 |
response = connection.recv_bytes(256) # reject large message |
| 1005 |
if response != _WELCOME: |
| 1006 |
raise AuthenticationError('digest sent was rejected') |
| 1007 |
|
| 1008 |
# |
| 1009 |
# Support for using xmlrpclib for serialization |
| 1010 |
# |
| 1011 |
|
| 1012 |
class ConnectionWrapper(object): |
| 1013 |
def __init__(self, conn, dumps, loads): |
| 1014 |
self._conn = conn |
| 1015 |
self._dumps = dumps |
| 1016 |
self._loads = loads |
| 1017 |
for attr in ('fileno', 'close', 'poll', 'recv_bytes', 'send_bytes'): |
| 1018 |
obj = getattr(conn, attr) |
| 1019 |
setattr(self, attr, obj) |
| 1020 |
def send(self, obj): |
| 1021 |
s = self._dumps(obj) |
| 1022 |
self._conn.send_bytes(s) |
| 1023 |
def recv(self): |
| 1024 |
s = self._conn.recv_bytes() |
| 1025 |
return self._loads(s) |
| 1026 |
|
| 1027 |
def _xml_dumps(obj): |
| 1028 |
return xmlrpclib.dumps((obj,), None, None, None, 1).encode('utf-8') |
| 1029 |
|
| 1030 |
def _xml_loads(s): |
| 1031 |
(obj,), method = xmlrpclib.loads(s.decode('utf-8')) |
| 1032 |
return obj |
| 1033 |
|
| 1034 |
class XmlListener(Listener): |
| 1035 |
def accept(self): |
| 1036 |
global xmlrpclib |
| 1037 |
import xmlrpc.client as xmlrpclib |
| 1038 |
obj = Listener.accept(self) |
| 1039 |
return ConnectionWrapper(obj, _xml_dumps, _xml_loads) |
| 1040 |
|
| 1041 |
def XmlClient(*args, **kwds): |
| 1042 |
global xmlrpclib |
| 1043 |
import xmlrpc.client as xmlrpclib |
| 1044 |
return ConnectionWrapper(Client(*args, **kwds), _xml_dumps, _xml_loads) |
| 1045 |
|
| 1046 |
# |
| 1047 |
# Wait |
| 1048 |
# |
| 1049 |
|
| 1050 |
if sys.platform == 'win32': |
| 1051 |
|
| 1052 |
def _exhaustive_wait(handles, timeout): |
| 1053 |
# Return ALL handles which are currently signalled. (Only |
| 1054 |
# returning the first signalled might create starvation issues.) |
| 1055 |
L = list(handles) |
| 1056 |
ready = [] |
| 1057 |
# Windows limits WaitForMultipleObjects at 64 handles, and we use a |
| 1058 |
# few for synchronisation, so we switch to batched waits at 60. |
| 1059 |
if len(L) > 60: |
| 1060 |
try: |
| 1061 |
res = _winapi.BatchedWaitForMultipleObjects(L, False, timeout) |
| 1062 |
except TimeoutError: |
| 1063 |
return [] |
| 1064 |
ready.extend(L[i] for i in res) |
| 1065 |
if res: |
| 1066 |
L = [h for i, h in enumerate(L) if i > res[0] & i not in res] |
| 1067 |
timeout = 0 |
| 1068 |
while L: |
| 1069 |
short_L = L[:60] if len(L) > 60 else L |
| 1070 |
res = _winapi.WaitForMultipleObjects(short_L, False, timeout) |
| 1071 |
if res == WAIT_TIMEOUT: |
| 1072 |
break |
| 1073 |
elif WAIT_OBJECT_0 <= res < WAIT_OBJECT_0 + len(L): |
| 1074 |
res -= WAIT_OBJECT_0 |
| 1075 |
elif WAIT_ABANDONED_0 <= res < WAIT_ABANDONED_0 + len(L): |
| 1076 |
res -= WAIT_ABANDONED_0 |
| 1077 |
else: |
| 1078 |
raise RuntimeError('Should not get here') |
| 1079 |
ready.append(L[res]) |
| 1080 |
L = L[res+1:] |
| 1081 |
timeout = 0 |
| 1082 |
return ready |
| 1083 |
|
| 1084 |
_ready_errors = {_winapi.ERROR_BROKEN_PIPE, _winapi.ERROR_NETNAME_DELETED} |
| 1085 |
|
| 1086 |
def wait(object_list, timeout=None): |
| 1087 |
''' |
| 1088 |
Wait till an object in object_list is ready/readable. |
| 1089 |
|
| 1090 |
Returns list of those objects in object_list which are ready/readable. |
| 1091 |
''' |
| 1092 |
if timeout is None: |
| 1093 |
timeout = INFINITE |
| 1094 |
elif timeout < 0: |
| 1095 |
timeout = 0 |
| 1096 |
else: |
| 1097 |
timeout = int(timeout * 1000 + 0.5) |
| 1098 |
|
| 1099 |
object_list = list(object_list) |
| 1100 |
waithandle_to_obj = {} |
| 1101 |
ov_list = [] |
| 1102 |
ready_objects = set() |
| 1103 |
ready_handles = set() |
| 1104 |
|
| 1105 |
try: |
| 1106 |
for o in object_list: |
| 1107 |
try: |
| 1108 |
fileno = getattr(o, 'fileno') |
| 1109 |
except AttributeError: |
| 1110 |
waithandle_to_obj[o.__index__()] = o |
| 1111 |
else: |
| 1112 |
# start an overlapped read of length zero |
| 1113 |
try: |
| 1114 |
ov, err = _winapi.ReadFile(fileno(), 0, True) |
| 1115 |
except OSError as e: |
| 1116 |
ov, err = None, e.winerror |
| 1117 |
if err not in _ready_errors: |
| 1118 |
raise |
| 1119 |
if err == _winapi.ERROR_IO_PENDING: |
| 1120 |
ov_list.append(ov) |
| 1121 |
waithandle_to_obj[ov.event] = o |
| 1122 |
else: |
| 1123 |
# If o.fileno() is an overlapped pipe handle and |
| 1124 |
# err == 0 then there is a zero length message |
| 1125 |
# in the pipe, but it HAS NOT been consumed... |
| 1126 |
if ov and sys.getwindowsversion()[:2] >= (6, 2): |
| 1127 |
# ... except on Windows 8 and later, where |
| 1128 |
# the message HAS been consumed. |
| 1129 |
try: |
| 1130 |
_, err = ov.GetOverlappedResult(False) |
| 1131 |
except OSError as e: |
| 1132 |
err = e.winerror |
| 1133 |
if not err and hasattr(o, '_got_empty_message'): |
| 1134 |
o._got_empty_message = True |
| 1135 |
ready_objects.add(o) |
| 1136 |
timeout = 0 |
| 1137 |
|
| 1138 |
ready_handles = _exhaustive_wait(waithandle_to_obj.keys(), timeout) |
| 1139 |
finally: |
| 1140 |
# request that overlapped reads stop |
| 1141 |
for ov in ov_list: |
| 1142 |
ov.cancel() |
| 1143 |
|
| 1144 |
# wait for all overlapped reads to stop |
| 1145 |
for ov in ov_list: |
| 1146 |
try: |
| 1147 |
_, err = ov.GetOverlappedResult(True) |
| 1148 |
except OSError as e: |
| 1149 |
err = e.winerror |
| 1150 |
if err not in _ready_errors: |
| 1151 |
raise |
| 1152 |
if err != _winapi.ERROR_OPERATION_ABORTED: |
| 1153 |
o = waithandle_to_obj[ov.event] |
| 1154 |
ready_objects.add(o) |
| 1155 |
if err == 0: |
| 1156 |
# If o.fileno() is an overlapped pipe handle then |
| 1157 |
# a zero length message HAS been consumed. |
| 1158 |
if hasattr(o, '_got_empty_message'): |
| 1159 |
o._got_empty_message = True |
| 1160 |
|
| 1161 |
ready_objects.update(waithandle_to_obj[h] for h in ready_handles) |
| 1162 |
return [o for o in object_list if o in ready_objects] |
| 1163 |
|
| 1164 |
else: |
| 1165 |
|
| 1166 |
import selectors |
| 1167 |
|
| 1168 |
# poll/select have the advantage of not requiring any extra file |
| 1169 |
# descriptor, contrarily to epoll/kqueue (also, they require a single |
| 1170 |
# syscall). |
| 1171 |
if hasattr(selectors, 'PollSelector'): |
| 1172 |
_WaitSelector = selectors.PollSelector |
| 1173 |
else: |
| 1174 |
_WaitSelector = selectors.SelectSelector |
| 1175 |
|
| 1176 |
def wait(object_list, timeout=None): |
| 1177 |
''' |
| 1178 |
Wait till an object in object_list is ready/readable. |
| 1179 |
|
| 1180 |
Returns list of those objects in object_list which are ready/readable. |
| 1181 |
''' |
| 1182 |
with _WaitSelector() as selector: |
| 1183 |
for obj in object_list: |
| 1184 |
selector.register(obj, selectors.EVENT_READ) |
| 1185 |
|
| 1186 |
if timeout is not None: |
| 1187 |
deadline = time.monotonic() + timeout |
| 1188 |
|
| 1189 |
while True: |
| 1190 |
ready = selector.select(timeout) |
| 1191 |
if ready: |
| 1192 |
return [key.fileobj for (key, events) in ready] |
| 1193 |
else: |
| 1194 |
if timeout is not None: |
| 1195 |
timeout = deadline - time.monotonic() |
| 1196 |
if timeout < 0: |
| 1197 |
return ready |
| 1198 |
|
| 1199 |
# |
| 1200 |
# Make connection and socket objects shareable if possible |
| 1201 |
# |
| 1202 |
|
| 1203 |
if sys.platform == 'win32': |
| 1204 |
def reduce_connection(conn): |
| 1205 |
handle = conn.fileno() |
| 1206 |
with socket.fromfd(handle, socket.AF_INET, socket.SOCK_STREAM) as s: |
| 1207 |
from . import resource_sharer |
| 1208 |
ds = resource_sharer.DupSocket(s) |
| 1209 |
return rebuild_connection, (ds, conn.readable, conn.writable) |
| 1210 |
def rebuild_connection(ds, readable, writable): |
| 1211 |
sock = ds.detach() |
| 1212 |
return Connection(sock.detach(), readable, writable) |
| 1213 |
reduction.register(Connection, reduce_connection) |
| 1214 |
|
| 1215 |
def reduce_pipe_connection(conn): |
| 1216 |
access = ((_winapi.FILE_GENERIC_READ if conn.readable else 0) | |
| 1217 |
(_winapi.FILE_GENERIC_WRITE if conn.writable else 0)) |
| 1218 |
dh = reduction.DupHandle(conn.fileno(), access) |
| 1219 |
return rebuild_pipe_connection, (dh, conn.readable, conn.writable) |
| 1220 |
def rebuild_pipe_connection(dh, readable, writable): |
| 1221 |
handle = dh.detach() |
| 1222 |
return PipeConnection(handle, readable, writable) |
| 1223 |
reduction.register(PipeConnection, reduce_pipe_connection) |
| 1224 |
|
| 1225 |
else: |
| 1226 |
def reduce_connection(conn): |
| 1227 |
df = reduction.DupFd(conn.fileno()) |
| 1228 |
return rebuild_connection, (df, conn.readable, conn.writable) |
| 1229 |
def rebuild_connection(df, readable, writable): |
| 1230 |
fd = df.detach() |
| 1231 |
return Connection(fd, readable, writable) |
| 1232 |
reduction.register(Connection, reduce_connection) |