Debian Codegraph @

Lib/multiprocessing/connection.py

raw
1 #
2 # A higher level module for using sockets (or Windows named pipes)
3 #
4 # multiprocessing/connection.py
5 #
6 # Copyright (c) 2006-2008, R Oudkerk
7 # Licensed to PSF under a Contributor Agreement.
8 #
9
10 __all__ = [ 'Client', 'Listener', 'Pipe', 'wait' ]
11
12 import errno
13 import io
14 import os
15 import sys
16 import socket
17 import struct
18 import time
19 import tempfile
20 import itertools
21
22
23 from . import util
24
25 from . import AuthenticationError, BufferTooShort
26 from .context import reduction
27 _ForkingPickler = reduction.ForkingPickler
28
29 try:
30 import _multiprocessing
31 import _winapi
32 from _winapi import WAIT_OBJECT_0, WAIT_ABANDONED_0, WAIT_TIMEOUT, INFINITE
33 except ImportError:
34 if sys.platform == 'win32':
35 raise
36 _winapi = None
37
38 #
39 #
40 #
41
42 BUFSIZE = 8192
43 # A very generous timeout when it comes to local connections...
44 CONNECTION_TIMEOUT = 20.
45
46 _mmap_counter = itertools.count()
47 _MAX_PIPE_ATTEMPTS = 100
48
49 default_family = 'AF_INET'
50 families = ['AF_INET']
51
52 if hasattr(socket, 'AF_UNIX'):
53 default_family = 'AF_UNIX'
54 families += ['AF_UNIX']
55
56 if sys.platform == 'win32':
57 default_family = 'AF_PIPE'
58 families += ['AF_PIPE']
59
60
61 def _init_timeout(timeout=CONNECTION_TIMEOUT):
62 return time.monotonic() + timeout
63
64 def _check_timeout(t):
65 return time.monotonic() > t
66
67 #
68 #
69 #
70
71 def arbitrary_address(family):
72 '''
73 Return an arbitrary free address for the given family
74 '''
75 if family == 'AF_INET':
76 return ('localhost', 0)
77 elif family == 'AF_UNIX':
78 # NOTE: util.get_temp_dir() is a 0o700 per-process directory. A
79 # mktemp-style ToC vs ToU concern is not important; bind() surfaces
80 # the extremely unlikely collision as EADDRINUSE.
81 return os.path.join(util.get_temp_dir(),
82 f'sock-{os.urandom(6).hex()}')
83 elif family == 'AF_PIPE':
84 return (r'\\.\pipe\pyc-%d-%d-%s' %
85 (os.getpid(), next(_mmap_counter), os.urandom(8).hex()))
86 else:
87 raise ValueError('unrecognized family')
88
89 def _validate_family(family):
90 '''
91 Checks if the family is valid for the current environment.
92 '''
93 if sys.platform != 'win32' and family == 'AF_PIPE':
94 raise ValueError('Family %s is not recognized.' % family)
95
96 if sys.platform == 'win32' and family == 'AF_UNIX':
97 # double check
98 if not hasattr(socket, family):
99 raise ValueError('Family %s is not recognized.' % family)
100
101 def address_type(address):
102 '''
103 Return the types of the address
104
105 This can be 'AF_INET', 'AF_UNIX', or 'AF_PIPE'
106 '''
107 if type(address) == tuple:
108 return 'AF_INET'
109 elif type(address) is str and address.startswith('\\\\'):
110 return 'AF_PIPE'
111 elif type(address) is str or util.is_abstract_socket_namespace(address):
112 return 'AF_UNIX'
113 else:
114 raise ValueError('address type of %r unrecognized' % address)
115
116 #
117 # Connection classes
118 #
119
120 class _ConnectionBase:
121 _handle = None
122
123 def __init__(self, handle, readable=True, writable=True):
124 handle = handle.__index__()
125 if handle < 0:
126 raise ValueError("invalid handle")
127 if not readable and not writable:
128 raise ValueError(
129 "at least one of `readable` and `writable` must be True")
130 self._handle = handle
131 self._readable = readable
132 self._writable = writable
133
134 # XXX should we use util.Finalize instead of a __del__?
135
136 def __del__(self):
137 if self._handle is not None:
138 self._close()
139
140 def _check_closed(self):
141 if self._handle is None:
142 raise OSError("handle is closed")
143
144 def _check_readable(self):
145 if not self._readable:
146 raise OSError("connection is write-only")
147
148 def _check_writable(self):
149 if not self._writable:
150 raise OSError("connection is read-only")
151
152 def _bad_message_length(self):
153 if self._writable:
154 self._readable = False
155 else:
156 self.close()
157 raise OSError("bad message length")
158
159 @property
160 def closed(self):
161 """True if the connection is closed"""
162 return self._handle is None
163
164 @property
165 def readable(self):
166 """True if the connection is readable"""
167 return self._readable
168
169 @property
170 def writable(self):
171 """True if the connection is writable"""
172 return self._writable
173
174 def fileno(self):
175 """File descriptor or handle of the connection"""
176 self._check_closed()
177 return self._handle
178
179 def close(self):
180 """Close the connection"""
181 if self._handle is not None:
182 try:
183 self._close()
184 finally:
185 self._handle = None
186
187 def send_bytes(self, buf, offset=0, size=None):
188 """Send the bytes data from a bytes-like object"""
189 self._check_closed()
190 self._check_writable()
191 m = memoryview(buf)
192 if m.itemsize > 1:
193 m = m.cast('B')
194 n = m.nbytes
195 if offset < 0:
196 raise ValueError("offset is negative")
197 if n < offset:
198 raise ValueError("buffer length < offset")
199 if size is None:
200 size = n - offset
201 elif size < 0:
202 raise ValueError("size is negative")
203 elif offset + size > n:
204 raise ValueError("buffer length < offset + size")
205 self._send_bytes(m[offset:offset + size])
206
207 def send(self, obj):
208 """Send a (picklable) object"""
209 self._check_closed()
210 self._check_writable()
211 self._send_bytes(_ForkingPickler.dumps(obj))
212
213 def recv_bytes(self, maxlength=None):
214 """
215 Receive bytes data as a bytes object.
216 """
217 self._check_closed()
218 self._check_readable()
219 if maxlength is not None and maxlength < 0:
220 raise ValueError("negative maxlength")
221 buf = self._recv_bytes(maxlength)
222 if buf is None:
223 self._bad_message_length()
224 return buf.getvalue()
225
226 def recv_bytes_into(self, buf, offset=0):
227 """
228 Receive bytes data into a writeable bytes-like object.
229 Return the number of bytes read.
230 """
231 self._check_closed()
232 self._check_readable()
233 with memoryview(buf) as m:
234 # Get bytesize of arbitrary buffer
235 itemsize = m.itemsize
236 bytesize = itemsize * len(m)
237 if offset < 0:
238 raise ValueError("negative offset")
239 elif offset > bytesize:
240 raise ValueError("offset too large")
241 result = self._recv_bytes()
242 size = result.tell()
243 if bytesize < offset + size:
244 raise BufferTooShort(result.getvalue())
245 # Message can fit in dest
246 result.seek(0)
247 result.readinto(m[offset // itemsize :
248 (offset + size) // itemsize])
249 return size
250
251 def recv(self):
252 """Receive a (picklable) object"""
253 self._check_closed()
254 self._check_readable()
255 buf = self._recv_bytes()
256 return _ForkingPickler.loads(buf.getbuffer())
257
258 def poll(self, timeout=0.0):
259 """Whether there is any input available to be read"""
260 self._check_closed()
261 self._check_readable()
262 return self._poll(timeout)
263
264 def __enter__(self):
265 return self
266
267 def __exit__(self, exc_type, exc_value, exc_tb):
268 self.close()
269
270
271 if _winapi:
272
273 class PipeConnection(_ConnectionBase):
274 """
275 Connection class based on a Windows named pipe.
276 Overlapped I/O is used, so the handles must have been created
277 with FILE_FLAG_OVERLAPPED.
278 """
279 _got_empty_message = False
280 _send_ov = None
281
282 def _close(self, _CloseHandle=_winapi.CloseHandle):
283 ov = self._send_ov
284 if ov is not None:
285 # Interrupt WaitForMultipleObjects() in _send_bytes()
286 ov.cancel()
287 _CloseHandle(self._handle)
288
289 def _send_bytes(self, buf):
290 if self._send_ov is not None:
291 # A connection should only be used by a single thread
292 raise ValueError("concurrent send_bytes() calls "
293 "are not supported")
294 ov, err = _winapi.WriteFile(self._handle, buf, overlapped=True)
295 self._send_ov = ov
296 try:
297 if err == _winapi.ERROR_IO_PENDING:
298 waitres = _winapi.WaitForMultipleObjects(
299 [ov.event], False, INFINITE)
300 assert waitres == WAIT_OBJECT_0
301 except:
302 ov.cancel()
303 raise
304 finally:
305 self._send_ov = None
306 nwritten, err = ov.GetOverlappedResult(True)
307 if err == _winapi.ERROR_OPERATION_ABORTED:
308 # close() was called by another thread while
309 # WaitForMultipleObjects() was waiting for the overlapped
310 # operation.
311 raise OSError(errno.EPIPE, "handle is closed")
312 assert err == 0
313 assert nwritten == len(buf)
314
315 def _recv_bytes(self, maxsize=None):
316 if self._got_empty_message:
317 self._got_empty_message = False
318 return io.BytesIO()
319 else:
320 bsize = 128 if maxsize is None else min(maxsize, 128)
321 try:
322 ov, err = _winapi.ReadFile(self._handle, bsize,
323 overlapped=True)
324 try:
325 if err == _winapi.ERROR_IO_PENDING:
326 waitres = _winapi.WaitForMultipleObjects(
327 [ov.event], False, INFINITE)
328 assert waitres == WAIT_OBJECT_0
329 except:
330 ov.cancel()
331 raise
332 finally:
333 nread, err = ov.GetOverlappedResult(True)
334 if err == 0:
335 f = io.BytesIO()
336 f.write(ov.getbuffer())
337 return f
338 elif err == _winapi.ERROR_MORE_DATA:
339 return self._get_more_data(ov, maxsize)
340 except OSError as e:
341 if e.winerror == _winapi.ERROR_BROKEN_PIPE:
342 raise EOFError
343 else:
344 raise
345 raise RuntimeError("shouldn't get here; expected KeyboardInterrupt")
346
347 def _poll(self, timeout):
348 if (self._got_empty_message or
349 _winapi.PeekNamedPipe(self._handle)[0] != 0):
350 return True
351 return bool(wait([self], timeout))
352
353 def _get_more_data(self, ov, maxsize):
354 buf = ov.getbuffer()
355 f = io.BytesIO()
356 f.write(buf)
357 left = _winapi.PeekNamedPipe(self._handle)[1]
358 assert left > 0
359 if maxsize is not None and len(buf) + left > maxsize:
360 self._bad_message_length()
361 ov, err = _winapi.ReadFile(self._handle, left, overlapped=True)
362 rbytes, err = ov.GetOverlappedResult(True)
363 assert err == 0
364 assert rbytes == left
365 f.write(ov.getbuffer())
366 return f
367
368
369 class Connection(_ConnectionBase):
370 """
371 Connection class based on an arbitrary file descriptor (Unix only), or
372 a socket handle (Windows).
373 """
374
375 if _winapi:
376 def _close(self, _close=_multiprocessing.closesocket):
377 _close(self._handle)
378 _write = _multiprocessing.send
379 _read = _multiprocessing.recv
380 else:
381 def _close(self, _close=os.close):
382 _close(self._handle)
383 _write = os.write
384 _read = os.read
385
386 def _send(self, buf, write=_write):
387 remaining = len(buf)
388 while True:
389 n = write(self._handle, buf)
390 remaining -= n
391 if remaining == 0:
392 break
393 buf = buf[n:]
394
395 def _recv(self, size, read=_read):
396 buf = io.BytesIO()
397 handle = self._handle
398 remaining = size
399 while remaining > 0:
400 chunk = read(handle, remaining)
401 n = len(chunk)
402 if n == 0:
403 if remaining == size:
404 raise EOFError
405 else:
406 raise OSError("got end of file during message")
407 buf.write(chunk)
408 remaining -= n
409 return buf
410
411 def _send_bytes(self, buf):
412 n = len(buf)
413 if n > 0x7fffffff:
414 pre_header = struct.pack("!i", -1)
415 header = struct.pack("!Q", n)
416 self._send(pre_header)
417 self._send(header)
418 self._send(buf)
419 else:
420 # For wire compatibility with 3.7 and lower
421 header = struct.pack("!i", n)
422 if n > 16384:
423 # The payload is large so Nagle's algorithm won't be triggered
424 # and we'd better avoid the cost of concatenation.
425 self._send(header)
426 self._send(buf)
427 else:
428 # Issue #20540: concatenate before sending, to avoid delays due
429 # to Nagle's algorithm on a TCP socket.
430 # Also note we want to avoid sending a 0-length buffer separately,
431 # to avoid "broken pipe" errors if the other end closed the pipe.
432 self._send(header + buf)
433
434 def _recv_bytes(self, maxsize=None):
435 buf = self._recv(4)
436 size, = struct.unpack("!i", buf.getvalue())
437 if size == -1:
438 buf = self._recv(8)
439 size, = struct.unpack("!Q", buf.getvalue())
440 if maxsize is not None and size > maxsize:
441 return None
442 return self._recv(size)
443
444 def _poll(self, timeout):
445 r = wait([self], timeout)
446 return bool(r)
447
448
449 #
450 # Public functions
451 #
452
453 class Listener(object):
454 '''
455 Returns a listener object.
456
457 This is a wrapper for a bound socket which is 'listening' for
458 connections, or for a Windows named pipe.
459 '''
460 def __init__(self, address=None, family=None, backlog=1, authkey=None):
461 family = family or (address and address_type(address)) \
462 or default_family
463 _validate_family(family)
464 if authkey is not None and not isinstance(authkey, bytes):
465 raise TypeError('authkey should be a byte string')
466
467 if family == 'AF_PIPE':
468 if address:
469 self._listener = PipeListener(address, backlog)
470 else:
471 for attempts in itertools.count():
472 address = arbitrary_address(family)
473 try:
474 self._listener = PipeListener(address, backlog)
475 break
476 except OSError as e:
477 if attempts >= _MAX_PIPE_ATTEMPTS:
478 raise
479 if e.winerror not in (_winapi.ERROR_PIPE_BUSY,
480 _winapi.ERROR_ACCESS_DENIED):
481 raise
482 else:
483 address = address or arbitrary_address(family)
484 self._listener = SocketListener(address, family, backlog)
485
486 self._authkey = authkey
487
488 def accept(self):
489 '''
490 Accept a connection on the bound socket or named pipe of `self`.
491
492 Returns a `Connection` object.
493 '''
494 if self._listener is None:
495 raise OSError('listener is closed')
496
497 c = self._listener.accept()
498 if self._authkey is not None:
499 deliver_challenge(c, self._authkey)
500 answer_challenge(c, self._authkey)
501 return c
502
503 def close(self):
504 '''
505 Close the bound socket or named pipe of `self`.
506 '''
507 listener = self._listener
508 if listener is not None:
509 self._listener = None
510 listener.close()
511
512 @property
513 def address(self):
514 return self._listener._address
515
516 @property
517 def last_accepted(self):
518 return self._listener._last_accepted
519
520 def __enter__(self):
521 return self
522
523 def __exit__(self, exc_type, exc_value, exc_tb):
524 self.close()
525
526
527 def Client(address, family=None, authkey=None):
528 '''
529 Returns a connection to the address of a `Listener`
530 '''
531 family = family or address_type(address)
532 _validate_family(family)
533 if family == 'AF_PIPE':
534 c = PipeClient(address)
535 else:
536 c = SocketClient(address)
537
538 if authkey is not None and not isinstance(authkey, bytes):
539 raise TypeError('authkey should be a byte string')
540
541 if authkey is not None:
542 answer_challenge(c, authkey)
543 deliver_challenge(c, authkey)
544
545 return c
546
547
548 if sys.platform != 'win32':
549
550 def Pipe(duplex=True):
551 '''
552 Returns pair of connection objects at either end of a pipe
553 '''
554 if duplex:
555 s1, s2 = socket.socketpair()
556 s1.setblocking(True)
557 s2.setblocking(True)
558 c1 = Connection(s1.detach())
559 c2 = Connection(s2.detach())
560 else:
561 fd1, fd2 = os.pipe()
562 c1 = Connection(fd1, writable=False)
563 c2 = Connection(fd2, readable=False)
564
565 return c1, c2
566
567 else:
568
569 def Pipe(duplex=True):
570 '''
571 Returns pair of connection objects at either end of a pipe
572 '''
573 if duplex:
574 openmode = _winapi.PIPE_ACCESS_DUPLEX
575 access = _winapi.GENERIC_READ | _winapi.GENERIC_WRITE
576 obsize, ibsize = BUFSIZE, BUFSIZE
577 else:
578 openmode = _winapi.PIPE_ACCESS_INBOUND
579 access = _winapi.GENERIC_WRITE
580 obsize, ibsize = 0, BUFSIZE
581
582 for attempts in itertools.count():
583 address = arbitrary_address('AF_PIPE')
584 try:
585 h1 = _winapi.CreateNamedPipe(
586 address, openmode | _winapi.FILE_FLAG_OVERLAPPED |
587 _winapi.FILE_FLAG_FIRST_PIPE_INSTANCE,
588 _winapi.PIPE_TYPE_MESSAGE | _winapi.PIPE_READMODE_MESSAGE |
589 _winapi.PIPE_WAIT,
590 1, obsize, ibsize, _winapi.NMPWAIT_WAIT_FOREVER,
591 # default security descriptor: the handle cannot be inherited
592 _winapi.NULL
593 )
594 break
595 except OSError as e:
596 if attempts >= _MAX_PIPE_ATTEMPTS:
597 raise
598 if e.winerror not in (_winapi.ERROR_PIPE_BUSY,
599 _winapi.ERROR_ACCESS_DENIED):
600 raise
601 h2 = _winapi.CreateFile(
602 address, access, 0, _winapi.NULL, _winapi.OPEN_EXISTING,
603 _winapi.FILE_FLAG_OVERLAPPED, _winapi.NULL
604 )
605 _winapi.SetNamedPipeHandleState(
606 h2, _winapi.PIPE_READMODE_MESSAGE, None, None
607 )
608
609 overlapped = _winapi.ConnectNamedPipe(h1, overlapped=True)
610 _, err = overlapped.GetOverlappedResult(True)
611 assert err == 0
612
613 c1 = PipeConnection(h1, writable=duplex)
614 c2 = PipeConnection(h2, readable=duplex)
615
616 return c1, c2
617
618 #
619 # Definitions for connections based on sockets
620 #
621
622 class SocketListener(object):
623 '''
624 Representation of a socket which is bound to an address and listening
625 '''
626 def __init__(self, address, family, backlog=1):
627 self._socket = socket.socket(getattr(socket, family))
628 try:
629 # SO_REUSEADDR has different semantics on Windows (issue #2550).
630 if os.name == 'posix':
631 self._socket.setsockopt(socket.SOL_SOCKET,
632 socket.SO_REUSEADDR, 1)
633 self._socket.setblocking(True)
634 self._socket.bind(address)
635 self._socket.listen(backlog)
636 self._address = self._socket.getsockname()
637 except OSError:
638 self._socket.close()
639 raise
640 self._family = family
641 self._last_accepted = None
642
643 if family == 'AF_UNIX' and not util.is_abstract_socket_namespace(address):
644 # Linux abstract socket namespaces do not need to be explicitly unlinked
645 self._unlink = util.Finalize(
646 self, os.unlink, args=(address,), exitpriority=0
647 )
648 else:
649 self._unlink = None
650
651 def accept(self):
652 s, self._last_accepted = self._socket.accept()
653 s.setblocking(True)
654 return Connection(s.detach())
655
656 def close(self):
657 try:
658 self._socket.close()
659 finally:
660 unlink = self._unlink
661 if unlink is not None:
662 self._unlink = None
663 unlink()
664
665
666 def SocketClient(address):
667 '''
668 Return a connection object connected to the socket given by `address`
669 '''
670 family = address_type(address)
671 with socket.socket( getattr(socket, family) ) as s:
672 s.setblocking(True)
673 s.connect(address)
674 return Connection(s.detach())
675
676 #
677 # Definitions for connections based on named pipes
678 #
679
680 if sys.platform == 'win32':
681
682 class PipeListener(object):
683 '''
684 Representation of a named pipe
685 '''
686 def __init__(self, address, backlog=None):
687 self._address = address
688 self._handle_queue = [self._new_handle(first=True)]
689
690 self._last_accepted = None
691 util.sub_debug('listener created with address=%r', self._address)
692 self.close = util.Finalize(
693 self, PipeListener._finalize_pipe_listener,
694 args=(self._handle_queue, self._address), exitpriority=0
695 )
696
697 def _new_handle(self, first=False):
698 flags = _winapi.PIPE_ACCESS_DUPLEX | _winapi.FILE_FLAG_OVERLAPPED
699 if first:
700 flags |= _winapi.FILE_FLAG_FIRST_PIPE_INSTANCE
701 return _winapi.CreateNamedPipe(
702 self._address, flags,
703 _winapi.PIPE_TYPE_MESSAGE | _winapi.PIPE_READMODE_MESSAGE |
704 _winapi.PIPE_WAIT,
705 _winapi.PIPE_UNLIMITED_INSTANCES, BUFSIZE, BUFSIZE,
706 _winapi.NMPWAIT_WAIT_FOREVER, _winapi.NULL
707 )
708
709 def accept(self):
710 self._handle_queue.append(self._new_handle())
711 handle = self._handle_queue.pop(0)
712 try:
713 ov = _winapi.ConnectNamedPipe(handle, overlapped=True)
714 except OSError as e:
715 if e.winerror != _winapi.ERROR_NO_DATA:
716 raise
717 # ERROR_NO_DATA can occur if a client has already connected,
718 # written data and then disconnected -- see Issue 14725.
719 else:
720 try:
721 res = _winapi.WaitForMultipleObjects(
722 [ov.event], False, INFINITE)
723 except:
724 ov.cancel()
725 _winapi.CloseHandle(handle)
726 raise
727 finally:
728 _, err = ov.GetOverlappedResult(True)
729 assert err == 0
730 return PipeConnection(handle)
731
732 @staticmethod
733 def _finalize_pipe_listener(queue, address):
734 util.sub_debug('closing listener with address=%r', address)
735 for handle in queue:
736 _winapi.CloseHandle(handle)
737
738 def PipeClient(address):
739 '''
740 Return a connection object connected to the pipe given by `address`
741 '''
742 t = _init_timeout()
743 while 1:
744 try:
745 _winapi.WaitNamedPipe(address, 1000)
746 h = _winapi.CreateFile(
747 address, _winapi.GENERIC_READ | _winapi.GENERIC_WRITE,
748 0, _winapi.NULL, _winapi.OPEN_EXISTING,
749 _winapi.FILE_FLAG_OVERLAPPED, _winapi.NULL
750 )
751 except OSError as e:
752 if e.winerror not in (_winapi.ERROR_SEM_TIMEOUT,
753 _winapi.ERROR_PIPE_BUSY) or _check_timeout(t):
754 raise
755 else:
756 break
757 else:
758 raise
759
760 _winapi.SetNamedPipeHandleState(
761 h, _winapi.PIPE_READMODE_MESSAGE, None, None
762 )
763 return PipeConnection(h)
764
765 #
766 # Authentication stuff
767 #
768
769 MESSAGE_LENGTH = 40 # MUST be > 20
770
771 _CHALLENGE = b'#CHALLENGE#'
772 _WELCOME = b'#WELCOME#'
773 _FAILURE = b'#FAILURE#'
774
775 # multiprocessing.connection Authentication Handshake Protocol Description
776 # (as documented for reference after reading the existing code)
777 # =============================================================================
778 #
779 # On Windows: native pipes with "overlapped IO" are used to send the bytes,
780 # instead of the length prefix SIZE scheme described below. (ie: the OS deals
781 # with message sizes for us)
782 #
783 # Protocol error behaviors:
784 #
785 # On POSIX, any failure to receive the length prefix into SIZE, for SIZE greater
786 # than the requested maxsize to receive, or receiving fewer than SIZE bytes
787 # results in the connection being closed and auth to fail.
788 #
789 # On Windows, receiving too few bytes is never a low level _recv_bytes read
790 # error, receiving too many will trigger an error only if receive maxsize
791 # value was larger than 128 OR the if the data arrived in smaller pieces.
792 #
793 # Serving side Client side
794 # ------------------------------ ---------------------------------------
795 # 0. Open a connection on the pipe.
796 # 1. Accept connection.
797 # 2. Random 20+ bytes -> MESSAGE
798 # Modern servers always send
799 # more than 20 bytes and include
800 # a {digest} prefix on it with
801 # their preferred HMAC digest.
802 # Legacy ones send ==20 bytes.
803 # 3. send 4 byte length (net order)
804 # prefix followed by:
805 # b'#CHALLENGE#' + MESSAGE
806 # 4. Receive 4 bytes, parse as network byte
807 # order integer. If it is -1, receive an
808 # additional 8 bytes, parse that as network
809 # byte order. The result is the length of
810 # the data that follows -> SIZE.
811 # 5. Receive min(SIZE, 256) bytes -> M1
812 # 6. Assert that M1 starts with:
813 # b'#CHALLENGE#'
814 # 7. Strip that prefix from M1 into -> M2
815 # 7.1. Parse M2: if it is exactly 20 bytes in
816 # length this indicates a legacy server
817 # supporting only HMAC-MD5. Otherwise the
818 # 7.2. preferred digest is looked up from an
819 # expected "{digest}" prefix on M2. No prefix
820 # or unsupported digest? <- AuthenticationError
821 # 7.3. Put divined algorithm name in -> D_NAME
822 # 8. Compute HMAC-D_NAME of AUTHKEY, M2 -> C_DIGEST
823 # 9. Send 4 byte length prefix (net order)
824 # followed by C_DIGEST bytes.
825 # 10. Receive 4 or 4+8 byte length
826 # prefix (#4 dance) -> SIZE.
827 # 11. Receive min(SIZE, 256) -> C_D.
828 # 11.1. Parse C_D: legacy servers
829 # accept it as is, "md5" -> D_NAME
830 # 11.2. modern servers check the length
831 # of C_D, IF it is 16 bytes?
832 # 11.2.1. "md5" -> D_NAME
833 # and skip to step 12.
834 # 11.3. longer? expect and parse a "{digest}"
835 # prefix into -> D_NAME.
836 # Strip the prefix and store remaining
837 # bytes in -> C_D.
838 # 11.4. Don't like D_NAME? <- AuthenticationError
839 # 12. Compute HMAC-D_NAME of AUTHKEY,
840 # MESSAGE into -> M_DIGEST.
841 # 13. Compare M_DIGEST == C_D:
842 # 14a: Match? Send length prefix &
843 # b'#WELCOME#'
844 # <- RETURN
845 # 14b: Mismatch? Send len prefix &
846 # b'#FAILURE#'
847 # <- CLOSE & AuthenticationError
848 # 15. Receive 4 or 4+8 byte length prefix (net
849 # order) again as in #4 into -> SIZE.
850 # 16. Receive min(SIZE, 256) bytes -> M3.
851 # 17. Compare M3 == b'#WELCOME#':
852 # 17a. Match? <- RETURN
853 # 17b. Mismatch? <- CLOSE & AuthenticationError
854 #
855 # If this RETURNed, the connection remains open: it has been authenticated.
856 #
857 # Length prefixes are used consistently. Even on the legacy protocol, this
858 # was good fortune and allowed us to evolve the protocol by using the length
859 # of the opening challenge or length of the returned digest as a signal as
860 # to which protocol the other end supports.
861
862 _ALLOWED_DIGESTS = frozenset(
863 {b'md5', b'sha256', b'sha384', b'sha3_256', b'sha3_384'})
864 _MAX_DIGEST_LEN = max(len(_) for _ in _ALLOWED_DIGESTS)
865
866 # Old hmac-md5 only server versions from Python <=3.11 sent a message of this
867 # length. It happens to not match the length of any supported digest so we can
868 # use a message of this length to indicate that we should work in backwards
869 # compatible md5-only mode without a {digest_name} prefix on our response.
870 _MD5ONLY_MESSAGE_LENGTH = 20
871 _MD5_DIGEST_LEN = 16
872 _LEGACY_LENGTHS = (_MD5ONLY_MESSAGE_LENGTH, _MD5_DIGEST_LEN)
873
874
875 def _get_digest_name_and_payload(message): # type: (bytes) -> tuple[str, bytes]
876 """Returns a digest name and the payload for a response hash.
877
878 If a legacy protocol is detected based on the message length
879 or contents the digest name returned will be empty to indicate
880 legacy mode where MD5 and no digest prefix should be sent.
881 """
882 # modern message format: b"{digest}payload" longer than 20 bytes
883 # legacy message format: 16 or 20 byte b"payload"
884 if len(message) in _LEGACY_LENGTHS:
885 # Either this was a legacy server challenge, or we're processing
886 # a reply from a legacy client that sent an unprefixed 16-byte
887 # HMAC-MD5 response. All messages using the modern protocol will
888 # be longer than either of these lengths.
889 return '', message
890 if (message.startswith(b'{') and
891 (curly := message.find(b'}', 1, _MAX_DIGEST_LEN+2)) > 0):
892 digest = message[1:curly]
893 if digest in _ALLOWED_DIGESTS:
894 payload = message[curly+1:]
895 return digest.decode('ascii'), payload
896 raise AuthenticationError(
897 'unsupported message length, missing digest prefix, '
898 f'or unsupported digest: {message=}')
899
900
901 def _create_response(authkey, message):
902 """Create a MAC based on authkey and message
903
904 The MAC algorithm defaults to HMAC-MD5, unless MD5 is not available or
905 the message has a '{digest_name}' prefix. For legacy HMAC-MD5, the response
906 is the raw MAC, otherwise the response is prefixed with '{digest_name}',
907 e.g. b'{sha256}abcdefg...'
908
909 Note: The MAC protects the entire message including the digest_name prefix.
910 """
911 import hmac
912 digest_name = _get_digest_name_and_payload(message)[0]
913 # The MAC protects the entire message: digest header and payload.
914 if not digest_name:
915 # Legacy server without a {digest} prefix on message.
916 # Generate a legacy non-prefixed HMAC-MD5 reply.
917 try:
918 return hmac.new(authkey, message, 'md5').digest()
919 except ValueError:
920 # HMAC-MD5 is not available (FIPS mode?), fall back to
921 # HMAC-SHA2-256 modern protocol. The legacy server probably
922 # doesn't support it and will reject us anyways. :shrug:
923 digest_name = 'sha256'
924 # Modern protocol, indicate the digest used in the reply.
925 response = hmac.new(authkey, message, digest_name).digest()
926 return b'{%s}%s' % (digest_name.encode('ascii'), response)
927
928
929 def _verify_challenge(authkey, message, response):
930 """Verify MAC challenge
931
932 If our message did not include a digest_name prefix, the client is allowed
933 to select a stronger digest_name from _ALLOWED_DIGESTS.
934
935 In case our message is prefixed, a client cannot downgrade to a weaker
936 algorithm, because the MAC is calculated over the entire message
937 including the '{digest_name}' prefix.
938 """
939 import hmac
940 response_digest, response_mac = _get_digest_name_and_payload(response)
941 response_digest = response_digest or 'md5'
942 try:
943 expected = hmac.new(authkey, message, response_digest).digest()
944 except ValueError:
945 raise AuthenticationError(f'{response_digest=} unsupported')
946 if len(expected) != len(response_mac):
947 raise AuthenticationError(
948 f'expected {response_digest!r} of length {len(expected)} '
949 f'got {len(response_mac)}')
950 if not hmac.compare_digest(expected, response_mac):
951 raise AuthenticationError('digest received was wrong')
952
953
954 def deliver_challenge(connection, authkey: bytes, digest_name='sha256'):
955 if not isinstance(authkey, bytes):
956 raise ValueError(
957 "Authkey must be bytes, not {0!s}".format(type(authkey)))
958 assert MESSAGE_LENGTH > _MD5ONLY_MESSAGE_LENGTH, "protocol constraint"
959 message = os.urandom(MESSAGE_LENGTH)
960 message = b'{%s}%s' % (digest_name.encode('ascii'), message)
961 # Even when sending a challenge to a legacy client that does not support
962 # digest prefixes, they'll take the entire thing as a challenge and
963 # respond to it with a raw HMAC-MD5.
964 connection.send_bytes(_CHALLENGE + message)
965 response = connection.recv_bytes(256) # reject large message
966 try:
967 _verify_challenge(authkey, message, response)
968 except AuthenticationError:
969 connection.send_bytes(_FAILURE)
970 raise
971 else:
972 connection.send_bytes(_WELCOME)
973
974
975 def answer_challenge(connection, authkey: bytes):
976 if not isinstance(authkey, bytes):
977 raise ValueError(
978 "Authkey must be bytes, not {0!s}".format(type(authkey)))
979 message = connection.recv_bytes(256) # reject large message
980 if not message.startswith(_CHALLENGE):
981 raise AuthenticationError(
982 f'Protocol error, expected challenge: {message=}')
983 message = message[len(_CHALLENGE):]
984 if len(message) < _MD5ONLY_MESSAGE_LENGTH:
985 raise AuthenticationError(f'challenge too short: {len(message)} bytes')
986 digest = _create_response(authkey, message)
987 connection.send_bytes(digest)
988 response = connection.recv_bytes(256) # reject large message
989 if response != _WELCOME:
990 raise AuthenticationError('digest sent was rejected')
991
992 #
993 # Support for using xmlrpclib for serialization
994 #
995
996 class ConnectionWrapper(object):
997 def __init__(self, conn, dumps, loads):
998 self._conn = conn
999 self._dumps = dumps
1000 self._loads = loads
1001 for attr in ('fileno', 'close', 'poll', 'recv_bytes', 'send_bytes'):
1002 obj = getattr(conn, attr)
1003 setattr(self, attr, obj)
1004 def send(self, obj):
1005 s = self._dumps(obj)
1006 self._conn.send_bytes(s)
1007 def recv(self):
1008 s = self._conn.recv_bytes()
1009 return self._loads(s)
1010
1011 def _xml_dumps(obj):
1012 return xmlrpclib.dumps((obj,), None, None, None, 1).encode('utf-8')
1013
1014 def _xml_loads(s):
1015 (obj,), method = xmlrpclib.loads(s.decode('utf-8'))
1016 return obj
1017
1018 class XmlListener(Listener):
1019 def accept(self):
1020 global xmlrpclib
1021 import xmlrpc.client as xmlrpclib
1022 obj = Listener.accept(self)
1023 return ConnectionWrapper(obj, _xml_dumps, _xml_loads)
1024
1025 def XmlClient(*args, **kwds):
1026 global xmlrpclib
1027 import xmlrpc.client as xmlrpclib
1028 return ConnectionWrapper(Client(*args, **kwds), _xml_dumps, _xml_loads)
1029
1030 #
1031 # Wait
1032 #
1033
1034 if sys.platform == 'win32':
1035
1036 def _exhaustive_wait(handles, timeout):
1037 # Return ALL handles which are currently signalled. (Only
1038 # returning the first signalled might create starvation issues.)
1039 L = list(handles)
1040 ready = []
1041 # Windows limits WaitForMultipleObjects at 64 handles, and we use a
1042 # few for synchronisation, so we switch to batched waits at 60.
1043 if len(L) > 60:
1044 try:
1045 res = _winapi.BatchedWaitForMultipleObjects(L, False, timeout)
1046 except TimeoutError:
1047 return []
1048 ready.extend(L[i] for i in res)
1049 if res:
1050 L = [h for i, h in enumerate(L) if i > res[0] & i not in res]
1051 timeout = 0
1052 while L:
1053 short_L = L[:60] if len(L) > 60 else L
1054 res = _winapi.WaitForMultipleObjects(short_L, False, timeout)
1055 if res == WAIT_TIMEOUT:
1056 break
1057 elif WAIT_OBJECT_0 <= res < WAIT_OBJECT_0 + len(L):
1058 res -= WAIT_OBJECT_0
1059 elif WAIT_ABANDONED_0 <= res < WAIT_ABANDONED_0 + len(L):
1060 res -= WAIT_ABANDONED_0
1061 else:
1062 raise RuntimeError('Should not get here')
1063 ready.append(L[res])
1064 L = L[res+1:]
1065 timeout = 0
1066 return ready
1067
1068 _ready_errors = {_winapi.ERROR_BROKEN_PIPE, _winapi.ERROR_NETNAME_DELETED}
1069
1070 def wait(object_list, timeout=None):
1071 '''
1072 Wait till an object in object_list is ready/readable.
1073
1074 Returns list of those objects in object_list which are ready/readable.
1075 '''
1076 if timeout is None:
1077 timeout = INFINITE
1078 elif timeout < 0:
1079 timeout = 0
1080 else:
1081 timeout = int(timeout * 1000 + 0.5)
1082
1083 object_list = list(object_list)
1084 waithandle_to_obj = {}
1085 ov_list = []
1086 ready_objects = set()
1087 ready_handles = set()
1088
1089 try:
1090 for o in object_list:
1091 try:
1092 fileno = getattr(o, 'fileno')
1093 except AttributeError:
1094 waithandle_to_obj[o.__index__()] = o
1095 else:
1096 # start an overlapped read of length zero
1097 try:
1098 ov, err = _winapi.ReadFile(fileno(), 0, True)
1099 except OSError as e:
1100 ov, err = None, e.winerror
1101 if err not in _ready_errors:
1102 raise
1103 if err == _winapi.ERROR_IO_PENDING:
1104 ov_list.append(ov)
1105 waithandle_to_obj[ov.event] = o
1106 else:
1107 # If o.fileno() is an overlapped pipe handle and
1108 # err == 0 then there is a zero length message
1109 # in the pipe, but it HAS NOT been consumed...
1110 if ov and sys.getwindowsversion()[:2] >= (6, 2):
1111 # ... except on Windows 8 and later, where
1112 # the message HAS been consumed.
1113 try:
1114 _, err = ov.GetOverlappedResult(False)
1115 except OSError as e:
1116 err = e.winerror
1117 if not err and hasattr(o, '_got_empty_message'):
1118 o._got_empty_message = True
1119 ready_objects.add(o)
1120 timeout = 0
1121
1122 ready_handles = _exhaustive_wait(waithandle_to_obj.keys(), timeout)
1123 finally:
1124 # request that overlapped reads stop
1125 for ov in ov_list:
1126 ov.cancel()
1127
1128 # wait for all overlapped reads to stop
1129 for ov in ov_list:
1130 try:
1131 _, err = ov.GetOverlappedResult(True)
1132 except OSError as e:
1133 err = e.winerror
1134 if err not in _ready_errors:
1135 raise
1136 if err != _winapi.ERROR_OPERATION_ABORTED:
1137 o = waithandle_to_obj[ov.event]
1138 ready_objects.add(o)
1139 if err == 0:
1140 # If o.fileno() is an overlapped pipe handle then
1141 # a zero length message HAS been consumed.
1142 if hasattr(o, '_got_empty_message'):
1143 o._got_empty_message = True
1144
1145 ready_objects.update(waithandle_to_obj[h] for h in ready_handles)
1146 return [o for o in object_list if o in ready_objects]
1147
1148 else:
1149
1150 import selectors
1151
1152 # poll/select have the advantage of not requiring any extra file
1153 # descriptor, contrarily to epoll/kqueue (also, they require a single
1154 # syscall).
1155 if hasattr(selectors, 'PollSelector'):
1156 _WaitSelector = selectors.PollSelector
1157 else:
1158 _WaitSelector = selectors.SelectSelector
1159
1160 def wait(object_list, timeout=None):
1161 '''
1162 Wait till an object in object_list is ready/readable.
1163
1164 Returns list of those objects in object_list which are ready/readable.
1165 '''
1166 with _WaitSelector() as selector:
1167 for obj in object_list:
1168 selector.register(obj, selectors.EVENT_READ)
1169
1170 if timeout is not None:
1171 deadline = time.monotonic() + timeout
1172
1173 while True:
1174 ready = selector.select(timeout)
1175 if ready:
1176 return [key.fileobj for (key, events) in ready]
1177 else:
1178 if timeout is not None:
1179 timeout = deadline - time.monotonic()
1180 if timeout < 0:
1181 return ready
1182
1183 #
1184 # Make connection and socket objects shareable if possible
1185 #
1186
1187 if sys.platform == 'win32':
1188 def reduce_connection(conn):
1189 handle = conn.fileno()
1190 with socket.fromfd(handle, socket.AF_INET, socket.SOCK_STREAM) as s:
1191 from . import resource_sharer
1192 ds = resource_sharer.DupSocket(s)
1193 return rebuild_connection, (ds, conn.readable, conn.writable)
1194 def rebuild_connection(ds, readable, writable):
1195 sock = ds.detach()
1196 return Connection(sock.detach(), readable, writable)
1197 reduction.register(Connection, reduce_connection)
1198
1199 def reduce_pipe_connection(conn):
1200 access = ((_winapi.FILE_GENERIC_READ if conn.readable else 0) |
1201 (_winapi.FILE_GENERIC_WRITE if conn.writable else 0))
1202 dh = reduction.DupHandle(conn.fileno(), access)
1203 return rebuild_pipe_connection, (dh, conn.readable, conn.writable)
1204 def rebuild_pipe_connection(dh, readable, writable):
1205 handle = dh.detach()
1206 return PipeConnection(handle, readable, writable)
1207 reduction.register(PipeConnection, reduce_pipe_connection)
1208
1209 else:
1210 def reduce_connection(conn):
1211 df = reduction.DupFd(conn.fileno())
1212 return rebuild_connection, (df, conn.readable, conn.writable)
1213 def rebuild_connection(df, readable, writable):
1214 fd = df.detach()
1215 return Connection(fd, readable, writable)
1216 reduction.register(Connection, reduce_connection)