Debian Codegraph @

Lib/multiprocessing/connection.py

raw
1 #
2 # A higher level module for using sockets (or Windows named pipes)
3 #
4 # multiprocessing/connection.py
5 #
6 # Copyright (c) 2006-2008, R Oudkerk
7 # Licensed to PSF under a Contributor Agreement.
8 #
9
10 __all__ = [ 'Client', 'Listener', 'Pipe', 'wait' ]
11
12 import errno
13 import io
14 import itertools
15 import os
16 import sys
17 import socket
18 import struct
19 import time
20
21
22 from . import util
23
24 from . import AuthenticationError, BufferTooShort
25 from .context import reduction
26 _ForkingPickler = reduction.ForkingPickler
27
28 try:
29 import _multiprocessing
30 import _winapi
31 from _winapi import WAIT_OBJECT_0, WAIT_ABANDONED_0, WAIT_TIMEOUT, INFINITE
32 except ImportError:
33 if sys.platform == 'win32':
34 raise
35 _winapi = None
36
37 #
38 #
39 #
40
41 # 64 KiB is the default PIPE buffer size of most POSIX platforms.
42 BUFSIZE = 64 * 1024
43
44 # A very generous timeout when it comes to local connections...
45 CONNECTION_TIMEOUT = 20.
46
47 _mmap_counter = itertools.count()
48 _MAX_PIPE_ATTEMPTS = 100
49
50 default_family = 'AF_INET'
51 families = ['AF_INET']
52
53 if hasattr(socket, 'AF_UNIX'):
54 default_family = 'AF_UNIX'
55 families += ['AF_UNIX']
56
57 if sys.platform == 'win32':
58 default_family = 'AF_PIPE'
59 families += ['AF_PIPE']
60
61
62 def _init_timeout(timeout=CONNECTION_TIMEOUT):
63 return time.monotonic() + timeout
64
65 def _check_timeout(t):
66 return time.monotonic() > t
67
68 #
69 #
70 #
71
72 def arbitrary_address(family):
73 '''
74 Return an arbitrary free address for the given family
75 '''
76 if family == 'AF_INET':
77 return ('localhost', 0)
78 elif family == 'AF_UNIX':
79 # NOTE: util.get_temp_dir() is a 0o700 per-process directory. A
80 # mktemp-style ToC vs ToU concern is not important; bind() surfaces
81 # the extremely unlikely collision as EADDRINUSE.
82 return os.path.join(util.get_temp_dir(),
83 f'sock-{os.urandom(6).hex()}')
84 elif family == 'AF_PIPE':
85 return (r'\\.\pipe\pyc-%d-%d-%s' %
86 (os.getpid(), next(_mmap_counter), os.urandom(8).hex()))
87 else:
88 raise ValueError('unrecognized family')
89
90 def _validate_family(family):
91 '''
92 Checks if the family is valid for the current environment.
93 '''
94 if sys.platform != 'win32' and family == 'AF_PIPE':
95 raise ValueError('Family %s is not recognized.' % family)
96
97 if sys.platform == 'win32' and family == 'AF_UNIX':
98 # double check
99 if not hasattr(socket, family):
100 raise ValueError('Family %s is not recognized.' % family)
101
102 def address_type(address):
103 '''
104 Return the types of the address
105
106 This can be 'AF_INET', 'AF_UNIX', or 'AF_PIPE'
107 '''
108 if type(address) == tuple:
109 return 'AF_INET'
110 elif type(address) is str and address.startswith('\\\\'):
111 return 'AF_PIPE'
112 elif type(address) is str or util.is_abstract_socket_namespace(address):
113 return 'AF_UNIX'
114 else:
115 raise ValueError('address type of %r unrecognized' % address)
116
117 #
118 # Connection classes
119 #
120
121 class _ConnectionBase:
122 _handle = None
123
124 def __init__(self, handle, readable=True, writable=True):
125 handle = handle.__index__()
126 if handle < 0:
127 raise ValueError("invalid handle")
128 if not readable and not writable:
129 raise ValueError(
130 "at least one of `readable` and `writable` must be True")
131 self._handle = handle
132 self._readable = readable
133 self._writable = writable
134
135 # XXX should we use util.Finalize instead of a __del__?
136
137 def __del__(self):
138 if self._handle is not None:
139 self._close()
140
141 def _check_closed(self):
142 if self._handle is None:
143 raise OSError("handle is closed")
144
145 def _check_readable(self):
146 if not self._readable:
147 raise OSError("connection is write-only")
148
149 def _check_writable(self):
150 if not self._writable:
151 raise OSError("connection is read-only")
152
153 def _bad_message_length(self):
154 if self._writable:
155 self._readable = False
156 else:
157 self.close()
158 raise OSError("bad message length")
159
160 @property
161 def closed(self):
162 """True if the connection is closed"""
163 return self._handle is None
164
165 @property
166 def readable(self):
167 """True if the connection is readable"""
168 return self._readable
169
170 @property
171 def writable(self):
172 """True if the connection is writable"""
173 return self._writable
174
175 def fileno(self):
176 """File descriptor or handle of the connection"""
177 self._check_closed()
178 return self._handle
179
180 def close(self):
181 """Close the connection"""
182 if self._handle is not None:
183 try:
184 self._close()
185 finally:
186 self._handle = None
187
188 def _detach(self):
189 """Stop managing the underlying file descriptor or handle."""
190 self._handle = None
191
192 def send_bytes(self, buf, offset=0, size=None):
193 """Send the bytes data from a bytes-like object"""
194 self._check_closed()
195 self._check_writable()
196 m = memoryview(buf)
197 if m.itemsize > 1:
198 m = m.cast('B')
199 n = m.nbytes
200 if offset < 0:
201 raise ValueError("offset is negative")
202 if n < offset:
203 raise ValueError("buffer length < offset")
204 if size is None:
205 size = n - offset
206 elif size < 0:
207 raise ValueError("size is negative")
208 elif offset + size > n:
209 raise ValueError("buffer length < offset + size")
210 self._send_bytes(m[offset:offset + size])
211
212 def send(self, obj):
213 """Send a (picklable) object"""
214 self._check_closed()
215 self._check_writable()
216 self._send_bytes(_ForkingPickler.dumps(obj))
217
218 def recv_bytes(self, maxlength=None):
219 """
220 Receive bytes data as a bytes object.
221 """
222 self._check_closed()
223 self._check_readable()
224 if maxlength is not None and maxlength < 0:
225 raise ValueError("negative maxlength")
226 buf = self._recv_bytes(maxlength)
227 if buf is None:
228 self._bad_message_length()
229 return buf.getvalue()
230
231 def recv_bytes_into(self, buf, offset=0):
232 """
233 Receive bytes data into a writeable bytes-like object.
234 Return the number of bytes read.
235 """
236 self._check_closed()
237 self._check_readable()
238 with memoryview(buf) as m:
239 # Get bytesize of arbitrary buffer
240 itemsize = m.itemsize
241 bytesize = itemsize * len(m)
242 if offset < 0:
243 raise ValueError("negative offset")
244 elif offset > bytesize:
245 raise ValueError("offset too large")
246 result = self._recv_bytes()
247 size = result.tell()
248 if bytesize < offset + size:
249 raise BufferTooShort(result.getvalue())
250 # Message can fit in dest
251 result.seek(0)
252 result.readinto(m[offset // itemsize :
253 (offset + size) // itemsize])
254 return size
255
256 def recv(self):
257 """Receive a (picklable) object"""
258 self._check_closed()
259 self._check_readable()
260 buf = self._recv_bytes()
261 return _ForkingPickler.loads(buf.getbuffer())
262
263 def poll(self, timeout=0.0):
264 """Whether there is any input available to be read"""
265 self._check_closed()
266 self._check_readable()
267 return self._poll(timeout)
268
269 def __enter__(self):
270 return self
271
272 def __exit__(self, exc_type, exc_value, exc_tb):
273 self.close()
274
275
276 if _winapi:
277
278 class PipeConnection(_ConnectionBase):
279 """
280 Connection class based on a Windows named pipe.
281 Overlapped I/O is used, so the handles must have been created
282 with FILE_FLAG_OVERLAPPED.
283 """
284 _got_empty_message = False
285 _send_ov = None
286
287 def _close(self, _CloseHandle=_winapi.CloseHandle):
288 ov = self._send_ov
289 if ov is not None:
290 # Interrupt WaitForMultipleObjects() in _send_bytes()
291 ov.cancel()
292 _CloseHandle(self._handle)
293
294 def _send_bytes(self, buf):
295 if self._send_ov is not None:
296 # A connection should only be used by a single thread
297 raise ValueError("concurrent send_bytes() calls "
298 "are not supported")
299 ov, err = _winapi.WriteFile(self._handle, buf, overlapped=True)
300 self._send_ov = ov
301 try:
302 if err == _winapi.ERROR_IO_PENDING:
303 waitres = _winapi.WaitForMultipleObjects(
304 [ov.event], False, INFINITE)
305 assert waitres == WAIT_OBJECT_0
306 except:
307 ov.cancel()
308 raise
309 finally:
310 self._send_ov = None
311 nwritten, err = ov.GetOverlappedResult(True)
312 if err == _winapi.ERROR_OPERATION_ABORTED:
313 # close() was called by another thread while
314 # WaitForMultipleObjects() was waiting for the overlapped
315 # operation.
316 raise OSError(errno.EPIPE, "handle is closed")
317 assert err == 0
318 assert nwritten == len(buf)
319
320 def _recv_bytes(self, maxsize=None):
321 if self._got_empty_message:
322 self._got_empty_message = False
323 return io.BytesIO()
324 else:
325 bsize = 128 if maxsize is None else min(maxsize, 128)
326 try:
327 ov, err = _winapi.ReadFile(self._handle, bsize,
328 overlapped=True)
329
330 sentinel = object()
331 return_value = sentinel
332 try:
333 try:
334 if err == _winapi.ERROR_IO_PENDING:
335 waitres = _winapi.WaitForMultipleObjects(
336 [ov.event], False, INFINITE)
337 assert waitres == WAIT_OBJECT_0
338 except:
339 ov.cancel()
340 raise
341 finally:
342 nread, err = ov.GetOverlappedResult(True)
343 if err == 0:
344 f = io.BytesIO()
345 f.write(ov.getbuffer())
346 return_value = f
347 elif err == _winapi.ERROR_MORE_DATA:
348 return_value = self._get_more_data(ov, maxsize)
349 except:
350 if return_value is sentinel:
351 raise
352
353 if return_value is not sentinel:
354 return return_value
355 except OSError as e:
356 if e.winerror == _winapi.ERROR_BROKEN_PIPE:
357 raise EOFError
358 else:
359 raise
360 raise RuntimeError("shouldn't get here; expected KeyboardInterrupt")
361
362 def _poll(self, timeout):
363 if (self._got_empty_message or
364 _winapi.PeekNamedPipe(self._handle)[0] != 0):
365 return True
366 return bool(wait([self], timeout))
367
368 def _get_more_data(self, ov, maxsize):
369 buf = ov.getbuffer()
370 f = io.BytesIO()
371 f.write(buf)
372 left = _winapi.PeekNamedPipe(self._handle)[1]
373 assert left > 0
374 if maxsize is not None and len(buf) + left > maxsize:
375 self._bad_message_length()
376 ov, err = _winapi.ReadFile(self._handle, left, overlapped=True)
377 rbytes, err = ov.GetOverlappedResult(True)
378 assert err == 0
379 assert rbytes == left
380 f.write(ov.getbuffer())
381 return f
382
383
384 class Connection(_ConnectionBase):
385 """
386 Connection class based on an arbitrary file descriptor (Unix only), or
387 a socket handle (Windows).
388 """
389
390 if _winapi:
391 def _close(self, _close=_multiprocessing.closesocket):
392 _close(self._handle)
393 _write = _multiprocessing.send
394 _read = _multiprocessing.recv
395 else:
396 def _close(self, _close=os.close):
397 _close(self._handle)
398 _write = os.write
399 _read = os.read
400
401 def _send(self, buf, write=_write):
402 remaining = len(buf)
403 while True:
404 n = write(self._handle, buf)
405 remaining -= n
406 if remaining == 0:
407 break
408 buf = buf[n:]
409
410 def _recv(self, size, read=_read):
411 buf = io.BytesIO()
412 handle = self._handle
413 remaining = size
414 while remaining > 0:
415 to_read = min(BUFSIZE, remaining)
416 chunk = read(handle, to_read)
417 n = len(chunk)
418 if n == 0:
419 if remaining == size:
420 raise EOFError
421 else:
422 raise OSError("got end of file during message")
423 buf.write(chunk)
424 remaining -= n
425 return buf
426
427 def _send_bytes(self, buf):
428 n = len(buf)
429 if n > 0x7fffffff:
430 pre_header = struct.pack("!i", -1)
431 header = struct.pack("!Q", n)
432 self._send(pre_header)
433 self._send(header)
434 self._send(buf)
435 else:
436 # For wire compatibility with 3.7 and lower
437 header = struct.pack("!i", n)
438 if n > 16384:
439 # The payload is large so Nagle's algorithm won't be triggered
440 # and we'd better avoid the cost of concatenation.
441 self._send(header)
442 self._send(buf)
443 else:
444 # Issue #20540: concatenate before sending, to avoid delays due
445 # to Nagle's algorithm on a TCP socket.
446 # Also note we want to avoid sending a 0-length buffer separately,
447 # to avoid "broken pipe" errors if the other end closed the pipe.
448 self._send(header + buf)
449
450 def _recv_bytes(self, maxsize=None):
451 buf = self._recv(4)
452 size, = struct.unpack("!i", buf.getvalue())
453 if size == -1:
454 buf = self._recv(8)
455 size, = struct.unpack("!Q", buf.getvalue())
456 if maxsize is not None and size > maxsize:
457 return None
458 return self._recv(size)
459
460 def _poll(self, timeout):
461 r = wait([self], timeout)
462 return bool(r)
463
464
465 #
466 # Public functions
467 #
468
469 class Listener(object):
470 '''
471 Returns a listener object.
472
473 This is a wrapper for a bound socket which is 'listening' for
474 connections, or for a Windows named pipe.
475 '''
476 def __init__(self, address=None, family=None, backlog=1, authkey=None):
477 family = family or (address and address_type(address)) \
478 or default_family
479 _validate_family(family)
480 if authkey is not None and not isinstance(authkey, bytes):
481 raise TypeError('authkey should be a byte string')
482
483 if family == 'AF_PIPE':
484 if address:
485 self._listener = PipeListener(address, backlog)
486 else:
487 for attempts in itertools.count():
488 address = arbitrary_address(family)
489 try:
490 self._listener = PipeListener(address, backlog)
491 break
492 except OSError as e:
493 if attempts >= _MAX_PIPE_ATTEMPTS:
494 raise
495 if e.winerror not in (_winapi.ERROR_PIPE_BUSY,
496 _winapi.ERROR_ACCESS_DENIED):
497 raise
498 else:
499 address = address or arbitrary_address(family)
500 self._listener = SocketListener(address, family, backlog)
501
502 self._authkey = authkey
503
504 def accept(self):
505 '''
506 Accept a connection on the bound socket or named pipe of `self`.
507
508 Returns a `Connection` object.
509 '''
510 if self._listener is None:
511 raise OSError('listener is closed')
512
513 c = self._listener.accept()
514 if self._authkey is not None:
515 deliver_challenge(c, self._authkey)
516 answer_challenge(c, self._authkey)
517 return c
518
519 def close(self):
520 '''
521 Close the bound socket or named pipe of `self`.
522 '''
523 listener = self._listener
524 if listener is not None:
525 self._listener = None
526 listener.close()
527
528 @property
529 def address(self):
530 return self._listener._address
531
532 @property
533 def last_accepted(self):
534 return self._listener._last_accepted
535
536 def __enter__(self):
537 return self
538
539 def __exit__(self, exc_type, exc_value, exc_tb):
540 self.close()
541
542
543 def Client(address, family=None, authkey=None):
544 '''
545 Returns a connection to the address of a `Listener`
546 '''
547 family = family or address_type(address)
548 _validate_family(family)
549 if family == 'AF_PIPE':
550 c = PipeClient(address)
551 else:
552 c = SocketClient(address)
553
554 if authkey is not None and not isinstance(authkey, bytes):
555 raise TypeError('authkey should be a byte string')
556
557 if authkey is not None:
558 answer_challenge(c, authkey)
559 deliver_challenge(c, authkey)
560
561 return c
562
563
564 if sys.platform != 'win32':
565
566 def Pipe(duplex=True):
567 '''
568 Returns pair of connection objects at either end of a pipe
569 '''
570 if duplex:
571 s1, s2 = socket.socketpair()
572 s1.setblocking(True)
573 s2.setblocking(True)
574 c1 = Connection(s1.detach())
575 c2 = Connection(s2.detach())
576 else:
577 fd1, fd2 = os.pipe()
578 c1 = Connection(fd1, writable=False)
579 c2 = Connection(fd2, readable=False)
580
581 return c1, c2
582
583 else:
584
585 def Pipe(duplex=True):
586 '''
587 Returns pair of connection objects at either end of a pipe
588 '''
589 if duplex:
590 openmode = _winapi.PIPE_ACCESS_DUPLEX
591 access = _winapi.GENERIC_READ | _winapi.GENERIC_WRITE
592 obsize, ibsize = BUFSIZE, BUFSIZE
593 else:
594 openmode = _winapi.PIPE_ACCESS_INBOUND
595 access = _winapi.GENERIC_WRITE
596 obsize, ibsize = 0, BUFSIZE
597
598 for attempts in itertools.count():
599 address = arbitrary_address('AF_PIPE')
600 try:
601 h1 = _winapi.CreateNamedPipe(
602 address, openmode | _winapi.FILE_FLAG_OVERLAPPED |
603 _winapi.FILE_FLAG_FIRST_PIPE_INSTANCE,
604 _winapi.PIPE_TYPE_MESSAGE | _winapi.PIPE_READMODE_MESSAGE |
605 _winapi.PIPE_WAIT,
606 1, obsize, ibsize, _winapi.NMPWAIT_WAIT_FOREVER,
607 # default security descriptor: the handle cannot be inherited
608 _winapi.NULL
609 )
610 break
611 except OSError as e:
612 if attempts >= _MAX_PIPE_ATTEMPTS:
613 raise
614 if e.winerror not in (_winapi.ERROR_PIPE_BUSY,
615 _winapi.ERROR_ACCESS_DENIED):
616 raise
617 h2 = _winapi.CreateFile(
618 address, access, 0, _winapi.NULL, _winapi.OPEN_EXISTING,
619 _winapi.FILE_FLAG_OVERLAPPED, _winapi.NULL
620 )
621 _winapi.SetNamedPipeHandleState(
622 h2, _winapi.PIPE_READMODE_MESSAGE, None, None
623 )
624
625 overlapped = _winapi.ConnectNamedPipe(h1, overlapped=True)
626 _, err = overlapped.GetOverlappedResult(True)
627 assert err == 0
628
629 c1 = PipeConnection(h1, writable=duplex)
630 c2 = PipeConnection(h2, readable=duplex)
631
632 return c1, c2
633
634 #
635 # Definitions for connections based on sockets
636 #
637
638 class SocketListener(object):
639 '''
640 Representation of a socket which is bound to an address and listening
641 '''
642 def __init__(self, address, family, backlog=1):
643 self._socket = socket.socket(getattr(socket, family))
644 try:
645 # SO_REUSEADDR has different semantics on Windows (issue #2550).
646 if os.name == 'posix':
647 self._socket.setsockopt(socket.SOL_SOCKET,
648 socket.SO_REUSEADDR, 1)
649 self._socket.setblocking(True)
650 self._socket.bind(address)
651 self._socket.listen(backlog)
652 self._address = self._socket.getsockname()
653 except OSError:
654 self._socket.close()
655 raise
656 self._family = family
657 self._last_accepted = None
658
659 if family == 'AF_UNIX' and not util.is_abstract_socket_namespace(address):
660 # Linux abstract socket namespaces do not need to be explicitly unlinked
661 self._unlink = util.Finalize(
662 self, os.unlink, args=(address,), exitpriority=0
663 )
664 else:
665 self._unlink = None
666
667 def accept(self):
668 s, self._last_accepted = self._socket.accept()
669 s.setblocking(True)
670 return Connection(s.detach())
671
672 def close(self):
673 try:
674 self._socket.close()
675 finally:
676 unlink = self._unlink
677 if unlink is not None:
678 self._unlink = None
679 unlink()
680
681
682 def SocketClient(address):
683 '''
684 Return a connection object connected to the socket given by `address`
685 '''
686 family = address_type(address)
687 with socket.socket( getattr(socket, family) ) as s:
688 s.setblocking(True)
689 s.connect(address)
690 return Connection(s.detach())
691
692 #
693 # Definitions for connections based on named pipes
694 #
695
696 if sys.platform == 'win32':
697
698 class PipeListener(object):
699 '''
700 Representation of a named pipe
701 '''
702 def __init__(self, address, backlog=None):
703 self._address = address
704 self._handle_queue = [self._new_handle(first=True)]
705
706 self._last_accepted = None
707 util.sub_debug('listener created with address=%r', self._address)
708 self.close = util.Finalize(
709 self, PipeListener._finalize_pipe_listener,
710 args=(self._handle_queue, self._address), exitpriority=0
711 )
712
713 def _new_handle(self, first=False):
714 flags = _winapi.PIPE_ACCESS_DUPLEX | _winapi.FILE_FLAG_OVERLAPPED
715 if first:
716 flags |= _winapi.FILE_FLAG_FIRST_PIPE_INSTANCE
717 return _winapi.CreateNamedPipe(
718 self._address, flags,
719 _winapi.PIPE_TYPE_MESSAGE | _winapi.PIPE_READMODE_MESSAGE |
720 _winapi.PIPE_WAIT,
721 _winapi.PIPE_UNLIMITED_INSTANCES, BUFSIZE, BUFSIZE,
722 _winapi.NMPWAIT_WAIT_FOREVER, _winapi.NULL
723 )
724
725 def accept(self):
726 self._handle_queue.append(self._new_handle())
727 handle = self._handle_queue.pop(0)
728 try:
729 ov = _winapi.ConnectNamedPipe(handle, overlapped=True)
730 except OSError as e:
731 if e.winerror != _winapi.ERROR_NO_DATA:
732 raise
733 # ERROR_NO_DATA can occur if a client has already connected,
734 # written data and then disconnected -- see Issue 14725.
735 else:
736 try:
737 res = _winapi.WaitForMultipleObjects(
738 [ov.event], False, INFINITE)
739 except:
740 ov.cancel()
741 _winapi.CloseHandle(handle)
742 raise
743 finally:
744 _, err = ov.GetOverlappedResult(True)
745 assert err == 0
746 return PipeConnection(handle)
747
748 @staticmethod
749 def _finalize_pipe_listener(queue, address):
750 util.sub_debug('closing listener with address=%r', address)
751 for handle in queue:
752 _winapi.CloseHandle(handle)
753
754 def PipeClient(address):
755 '''
756 Return a connection object connected to the pipe given by `address`
757 '''
758 t = _init_timeout()
759 while 1:
760 try:
761 _winapi.WaitNamedPipe(address, 1000)
762 h = _winapi.CreateFile(
763 address, _winapi.GENERIC_READ | _winapi.GENERIC_WRITE,
764 0, _winapi.NULL, _winapi.OPEN_EXISTING,
765 _winapi.FILE_FLAG_OVERLAPPED, _winapi.NULL
766 )
767 except OSError as e:
768 if e.winerror not in (_winapi.ERROR_SEM_TIMEOUT,
769 _winapi.ERROR_PIPE_BUSY) or _check_timeout(t):
770 raise
771 else:
772 break
773 else:
774 raise
775
776 _winapi.SetNamedPipeHandleState(
777 h, _winapi.PIPE_READMODE_MESSAGE, None, None
778 )
779 return PipeConnection(h)
780
781 #
782 # Authentication stuff
783 #
784
785 MESSAGE_LENGTH = 40 # MUST be > 20
786
787 _CHALLENGE = b'#CHALLENGE#'
788 _WELCOME = b'#WELCOME#'
789 _FAILURE = b'#FAILURE#'
790
791 # multiprocessing.connection Authentication Handshake Protocol Description
792 # (as documented for reference after reading the existing code)
793 # =============================================================================
794 #
795 # On Windows: native pipes with "overlapped IO" are used to send the bytes,
796 # instead of the length prefix SIZE scheme described below. (ie: the OS deals
797 # with message sizes for us)
798 #
799 # Protocol error behaviors:
800 #
801 # On POSIX, any failure to receive the length prefix into SIZE, for SIZE greater
802 # than the requested maxsize to receive, or receiving fewer than SIZE bytes
803 # results in the connection being closed and auth to fail.
804 #
805 # On Windows, receiving too few bytes is never a low level _recv_bytes read
806 # error, receiving too many will trigger an error only if receive maxsize
807 # value was larger than 128 OR the if the data arrived in smaller pieces.
808 #
809 # Serving side Client side
810 # ------------------------------ ---------------------------------------
811 # 0. Open a connection on the pipe.
812 # 1. Accept connection.
813 # 2. Random 20+ bytes -> MESSAGE
814 # Modern servers always send
815 # more than 20 bytes and include
816 # a {digest} prefix on it with
817 # their preferred HMAC digest.
818 # Legacy ones send ==20 bytes.
819 # 3. send 4 byte length (net order)
820 # prefix followed by:
821 # b'#CHALLENGE#' + MESSAGE
822 # 4. Receive 4 bytes, parse as network byte
823 # order integer. If it is -1, receive an
824 # additional 8 bytes, parse that as network
825 # byte order. The result is the length of
826 # the data that follows -> SIZE.
827 # 5. Receive min(SIZE, 256) bytes -> M1
828 # 6. Assert that M1 starts with:
829 # b'#CHALLENGE#'
830 # 7. Strip that prefix from M1 into -> M2
831 # 7.1. Parse M2: if it is exactly 20 bytes in
832 # length this indicates a legacy server
833 # supporting only HMAC-MD5. Otherwise the
834 # 7.2. preferred digest is looked up from an
835 # expected "{digest}" prefix on M2. No prefix
836 # or unsupported digest? <- AuthenticationError
837 # 7.3. Put divined algorithm name in -> D_NAME
838 # 8. Compute HMAC-D_NAME of AUTHKEY, M2 -> C_DIGEST
839 # 9. Send 4 byte length prefix (net order)
840 # followed by C_DIGEST bytes.
841 # 10. Receive 4 or 4+8 byte length
842 # prefix (#4 dance) -> SIZE.
843 # 11. Receive min(SIZE, 256) -> C_D.
844 # 11.1. Parse C_D: legacy servers
845 # accept it as is, "md5" -> D_NAME
846 # 11.2. modern servers check the length
847 # of C_D, IF it is 16 bytes?
848 # 11.2.1. "md5" -> D_NAME
849 # and skip to step 12.
850 # 11.3. longer? expect and parse a "{digest}"
851 # prefix into -> D_NAME.
852 # Strip the prefix and store remaining
853 # bytes in -> C_D.
854 # 11.4. Don't like D_NAME? <- AuthenticationError
855 # 12. Compute HMAC-D_NAME of AUTHKEY,
856 # MESSAGE into -> M_DIGEST.
857 # 13. Compare M_DIGEST == C_D:
858 # 14a: Match? Send length prefix &
859 # b'#WELCOME#'
860 # <- RETURN
861 # 14b: Mismatch? Send len prefix &
862 # b'#FAILURE#'
863 # <- CLOSE & AuthenticationError
864 # 15. Receive 4 or 4+8 byte length prefix (net
865 # order) again as in #4 into -> SIZE.
866 # 16. Receive min(SIZE, 256) bytes -> M3.
867 # 17. Compare M3 == b'#WELCOME#':
868 # 17a. Match? <- RETURN
869 # 17b. Mismatch? <- CLOSE & AuthenticationError
870 #
871 # If this RETURNed, the connection remains open: it has been authenticated.
872 #
873 # Length prefixes are used consistently. Even on the legacy protocol, this
874 # was good fortune and allowed us to evolve the protocol by using the length
875 # of the opening challenge or length of the returned digest as a signal as
876 # to which protocol the other end supports.
877
878 _ALLOWED_DIGESTS = frozenset(
879 {b'md5', b'sha256', b'sha384', b'sha3_256', b'sha3_384'})
880 _MAX_DIGEST_LEN = max(len(_) for _ in _ALLOWED_DIGESTS)
881
882 # Old hmac-md5 only server versions from Python <=3.11 sent a message of this
883 # length. It happens to not match the length of any supported digest so we can
884 # use a message of this length to indicate that we should work in backwards
885 # compatible md5-only mode without a {digest_name} prefix on our response.
886 _MD5ONLY_MESSAGE_LENGTH = 20
887 _MD5_DIGEST_LEN = 16
888 _LEGACY_LENGTHS = (_MD5ONLY_MESSAGE_LENGTH, _MD5_DIGEST_LEN)
889
890
891 def _get_digest_name_and_payload(message): # type: (bytes) -> tuple[str, bytes]
892 """Returns a digest name and the payload for a response hash.
893
894 If a legacy protocol is detected based on the message length
895 or contents the digest name returned will be empty to indicate
896 legacy mode where MD5 and no digest prefix should be sent.
897 """
898 # modern message format: b"{digest}payload" longer than 20 bytes
899 # legacy message format: 16 or 20 byte b"payload"
900 if len(message) in _LEGACY_LENGTHS:
901 # Either this was a legacy server challenge, or we're processing
902 # a reply from a legacy client that sent an unprefixed 16-byte
903 # HMAC-MD5 response. All messages using the modern protocol will
904 # be longer than either of these lengths.
905 return '', message
906 if (message.startswith(b'{') and
907 (curly := message.find(b'}', 1, _MAX_DIGEST_LEN+2)) > 0):
908 digest = message[1:curly]
909 if digest in _ALLOWED_DIGESTS:
910 payload = message[curly+1:]
911 return digest.decode('ascii'), payload
912 raise AuthenticationError(
913 'unsupported message length, missing digest prefix, '
914 f'or unsupported digest: {message=}')
915
916
917 def _create_response(authkey, message):
918 """Create a MAC based on authkey and message
919
920 The MAC algorithm defaults to HMAC-MD5, unless MD5 is not available or
921 the message has a '{digest_name}' prefix. For legacy HMAC-MD5, the response
922 is the raw MAC, otherwise the response is prefixed with '{digest_name}',
923 e.g. b'{sha256}abcdefg...'
924
925 Note: The MAC protects the entire message including the digest_name prefix.
926 """
927 import hmac
928 digest_name = _get_digest_name_and_payload(message)[0]
929 # The MAC protects the entire message: digest header and payload.
930 if not digest_name:
931 # Legacy server without a {digest} prefix on message.
932 # Generate a legacy non-prefixed HMAC-MD5 reply.
933 try:
934 return hmac.new(authkey, message, 'md5').digest()
935 except ValueError:
936 # HMAC-MD5 is not available (FIPS mode?), fall back to
937 # HMAC-SHA2-256 modern protocol. The legacy server probably
938 # doesn't support it and will reject us anyways. :shrug:
939 digest_name = 'sha256'
940 # Modern protocol, indicate the digest used in the reply.
941 response = hmac.new(authkey, message, digest_name).digest()
942 return b'{%s}%s' % (digest_name.encode('ascii'), response)
943
944
945 def _verify_challenge(authkey, message, response):
946 """Verify MAC challenge
947
948 If our message did not include a digest_name prefix, the client is allowed
949 to select a stronger digest_name from _ALLOWED_DIGESTS.
950
951 In case our message is prefixed, a client cannot downgrade to a weaker
952 algorithm, because the MAC is calculated over the entire message
953 including the '{digest_name}' prefix.
954 """
955 import hmac
956 response_digest, response_mac = _get_digest_name_and_payload(response)
957 response_digest = response_digest or 'md5'
958 try:
959 expected = hmac.new(authkey, message, response_digest).digest()
960 except ValueError:
961 raise AuthenticationError(f'{response_digest=} unsupported')
962 if len(expected) != len(response_mac):
963 raise AuthenticationError(
964 f'expected {response_digest!r} of length {len(expected)} '
965 f'got {len(response_mac)}')
966 if not hmac.compare_digest(expected, response_mac):
967 raise AuthenticationError('digest received was wrong')
968
969
970 def deliver_challenge(connection, authkey: bytes, digest_name='sha256'):
971 if not isinstance(authkey, bytes):
972 raise ValueError(
973 "Authkey must be bytes, not {0!s}".format(type(authkey)))
974 assert MESSAGE_LENGTH > _MD5ONLY_MESSAGE_LENGTH, "protocol constraint"
975 message = os.urandom(MESSAGE_LENGTH)
976 message = b'{%s}%s' % (digest_name.encode('ascii'), message)
977 # Even when sending a challenge to a legacy client that does not support
978 # digest prefixes, they'll take the entire thing as a challenge and
979 # respond to it with a raw HMAC-MD5.
980 connection.send_bytes(_CHALLENGE + message)
981 response = connection.recv_bytes(256) # reject large message
982 try:
983 _verify_challenge(authkey, message, response)
984 except AuthenticationError:
985 connection.send_bytes(_FAILURE)
986 raise
987 else:
988 connection.send_bytes(_WELCOME)
989
990
991 def answer_challenge(connection, authkey: bytes):
992 if not isinstance(authkey, bytes):
993 raise ValueError(
994 "Authkey must be bytes, not {0!s}".format(type(authkey)))
995 message = connection.recv_bytes(256) # reject large message
996 if not message.startswith(_CHALLENGE):
997 raise AuthenticationError(
998 f'Protocol error, expected challenge: {message=}')
999 message = message[len(_CHALLENGE):]
1000 if len(message) < _MD5ONLY_MESSAGE_LENGTH:
1001 raise AuthenticationError(f'challenge too short: {len(message)} bytes')
1002 digest = _create_response(authkey, message)
1003 connection.send_bytes(digest)
1004 response = connection.recv_bytes(256) # reject large message
1005 if response != _WELCOME:
1006 raise AuthenticationError('digest sent was rejected')
1007
1008 #
1009 # Support for using xmlrpclib for serialization
1010 #
1011
1012 class ConnectionWrapper(object):
1013 def __init__(self, conn, dumps, loads):
1014 self._conn = conn
1015 self._dumps = dumps
1016 self._loads = loads
1017 for attr in ('fileno', 'close', 'poll', 'recv_bytes', 'send_bytes'):
1018 obj = getattr(conn, attr)
1019 setattr(self, attr, obj)
1020 def send(self, obj):
1021 s = self._dumps(obj)
1022 self._conn.send_bytes(s)
1023 def recv(self):
1024 s = self._conn.recv_bytes()
1025 return self._loads(s)
1026
1027 def _xml_dumps(obj):
1028 return xmlrpclib.dumps((obj,), None, None, None, 1).encode('utf-8')
1029
1030 def _xml_loads(s):
1031 (obj,), method = xmlrpclib.loads(s.decode('utf-8'))
1032 return obj
1033
1034 class XmlListener(Listener):
1035 def accept(self):
1036 global xmlrpclib
1037 import xmlrpc.client as xmlrpclib
1038 obj = Listener.accept(self)
1039 return ConnectionWrapper(obj, _xml_dumps, _xml_loads)
1040
1041 def XmlClient(*args, **kwds):
1042 global xmlrpclib
1043 import xmlrpc.client as xmlrpclib
1044 return ConnectionWrapper(Client(*args, **kwds), _xml_dumps, _xml_loads)
1045
1046 #
1047 # Wait
1048 #
1049
1050 if sys.platform == 'win32':
1051
1052 def _exhaustive_wait(handles, timeout):
1053 # Return ALL handles which are currently signalled. (Only
1054 # returning the first signalled might create starvation issues.)
1055 L = list(handles)
1056 ready = []
1057 # Windows limits WaitForMultipleObjects at 64 handles, and we use a
1058 # few for synchronisation, so we switch to batched waits at 60.
1059 if len(L) > 60:
1060 try:
1061 res = _winapi.BatchedWaitForMultipleObjects(L, False, timeout)
1062 except TimeoutError:
1063 return []
1064 ready.extend(L[i] for i in res)
1065 if res:
1066 L = [h for i, h in enumerate(L) if i > res[0] & i not in res]
1067 timeout = 0
1068 while L:
1069 short_L = L[:60] if len(L) > 60 else L
1070 res = _winapi.WaitForMultipleObjects(short_L, False, timeout)
1071 if res == WAIT_TIMEOUT:
1072 break
1073 elif WAIT_OBJECT_0 <= res < WAIT_OBJECT_0 + len(L):
1074 res -= WAIT_OBJECT_0
1075 elif WAIT_ABANDONED_0 <= res < WAIT_ABANDONED_0 + len(L):
1076 res -= WAIT_ABANDONED_0
1077 else:
1078 raise RuntimeError('Should not get here')
1079 ready.append(L[res])
1080 L = L[res+1:]
1081 timeout = 0
1082 return ready
1083
1084 _ready_errors = {_winapi.ERROR_BROKEN_PIPE, _winapi.ERROR_NETNAME_DELETED}
1085
1086 def wait(object_list, timeout=None):
1087 '''
1088 Wait till an object in object_list is ready/readable.
1089
1090 Returns list of those objects in object_list which are ready/readable.
1091 '''
1092 if timeout is None:
1093 timeout = INFINITE
1094 elif timeout < 0:
1095 timeout = 0
1096 else:
1097 timeout = int(timeout * 1000 + 0.5)
1098
1099 object_list = list(object_list)
1100 waithandle_to_obj = {}
1101 ov_list = []
1102 ready_objects = set()
1103 ready_handles = set()
1104
1105 try:
1106 for o in object_list:
1107 try:
1108 fileno = getattr(o, 'fileno')
1109 except AttributeError:
1110 waithandle_to_obj[o.__index__()] = o
1111 else:
1112 # start an overlapped read of length zero
1113 try:
1114 ov, err = _winapi.ReadFile(fileno(), 0, True)
1115 except OSError as e:
1116 ov, err = None, e.winerror
1117 if err not in _ready_errors:
1118 raise
1119 if err == _winapi.ERROR_IO_PENDING:
1120 ov_list.append(ov)
1121 waithandle_to_obj[ov.event] = o
1122 else:
1123 # If o.fileno() is an overlapped pipe handle and
1124 # err == 0 then there is a zero length message
1125 # in the pipe, but it HAS NOT been consumed...
1126 if ov and sys.getwindowsversion()[:2] >= (6, 2):
1127 # ... except on Windows 8 and later, where
1128 # the message HAS been consumed.
1129 try:
1130 _, err = ov.GetOverlappedResult(False)
1131 except OSError as e:
1132 err = e.winerror
1133 if not err and hasattr(o, '_got_empty_message'):
1134 o._got_empty_message = True
1135 ready_objects.add(o)
1136 timeout = 0
1137
1138 ready_handles = _exhaustive_wait(waithandle_to_obj.keys(), timeout)
1139 finally:
1140 # request that overlapped reads stop
1141 for ov in ov_list:
1142 ov.cancel()
1143
1144 # wait for all overlapped reads to stop
1145 for ov in ov_list:
1146 try:
1147 _, err = ov.GetOverlappedResult(True)
1148 except OSError as e:
1149 err = e.winerror
1150 if err not in _ready_errors:
1151 raise
1152 if err != _winapi.ERROR_OPERATION_ABORTED:
1153 o = waithandle_to_obj[ov.event]
1154 ready_objects.add(o)
1155 if err == 0:
1156 # If o.fileno() is an overlapped pipe handle then
1157 # a zero length message HAS been consumed.
1158 if hasattr(o, '_got_empty_message'):
1159 o._got_empty_message = True
1160
1161 ready_objects.update(waithandle_to_obj[h] for h in ready_handles)
1162 return [o for o in object_list if o in ready_objects]
1163
1164 else:
1165
1166 import selectors
1167
1168 # poll/select have the advantage of not requiring any extra file
1169 # descriptor, contrarily to epoll/kqueue (also, they require a single
1170 # syscall).
1171 if hasattr(selectors, 'PollSelector'):
1172 _WaitSelector = selectors.PollSelector
1173 else:
1174 _WaitSelector = selectors.SelectSelector
1175
1176 def wait(object_list, timeout=None):
1177 '''
1178 Wait till an object in object_list is ready/readable.
1179
1180 Returns list of those objects in object_list which are ready/readable.
1181 '''
1182 with _WaitSelector() as selector:
1183 for obj in object_list:
1184 selector.register(obj, selectors.EVENT_READ)
1185
1186 if timeout is not None:
1187 deadline = time.monotonic() + timeout
1188
1189 while True:
1190 ready = selector.select(timeout)
1191 if ready:
1192 return [key.fileobj for (key, events) in ready]
1193 else:
1194 if timeout is not None:
1195 timeout = deadline - time.monotonic()
1196 if timeout < 0:
1197 return ready
1198
1199 #
1200 # Make connection and socket objects shareable if possible
1201 #
1202
1203 if sys.platform == 'win32':
1204 def reduce_connection(conn):
1205 handle = conn.fileno()
1206 with socket.fromfd(handle, socket.AF_INET, socket.SOCK_STREAM) as s:
1207 from . import resource_sharer
1208 ds = resource_sharer.DupSocket(s)
1209 return rebuild_connection, (ds, conn.readable, conn.writable)
1210 def rebuild_connection(ds, readable, writable):
1211 sock = ds.detach()
1212 return Connection(sock.detach(), readable, writable)
1213 reduction.register(Connection, reduce_connection)
1214
1215 def reduce_pipe_connection(conn):
1216 access = ((_winapi.FILE_GENERIC_READ if conn.readable else 0) |
1217 (_winapi.FILE_GENERIC_WRITE if conn.writable else 0))
1218 dh = reduction.DupHandle(conn.fileno(), access)
1219 return rebuild_pipe_connection, (dh, conn.readable, conn.writable)
1220 def rebuild_pipe_connection(dh, readable, writable):
1221 handle = dh.detach()
1222 return PipeConnection(handle, readable, writable)
1223 reduction.register(PipeConnection, reduce_pipe_connection)
1224
1225 else:
1226 def reduce_connection(conn):
1227 df = reduction.DupFd(conn.fileno())
1228 return rebuild_connection, (df, conn.readable, conn.writable)
1229 def rebuild_connection(df, readable, writable):
1230 fd = df.detach()
1231 return Connection(fd, readable, writable)
1232 reduction.register(Connection, reduce_connection)