Debian Codegraph @

Lib/asyncio/streams.py

raw
1 __all__ = (
2 'StreamReader', 'StreamWriter', 'StreamReaderProtocol',
3 'open_connection', 'start_server')
4
5 import collections
6 import socket
7 import sys
8 import warnings
9 import weakref
10
11 if hasattr(socket, 'AF_UNIX'):
12 __all__ += ('open_unix_connection', 'start_unix_server')
13
14 from . import coroutines
15 from . import events
16 from . import exceptions
17 from . import format_helpers
18 from . import protocols
19 from .log import logger
20 from .tasks import sleep
21
22
23 _DEFAULT_LIMIT = 2 ** 16 # 64 KiB
24
25
26 async def open_connection(host=None, port=None, *,
27 limit=_DEFAULT_LIMIT, **kwds):
28 """A wrapper for create_connection() returning a (reader, writer) pair.
29
30 The reader returned is a StreamReader instance; the writer is a
31 StreamWriter instance.
32
33 The arguments are all the usual arguments to create_connection()
34 except protocol_factory; most common are positional host and port,
35 with various optional keyword arguments following.
36
37 Additional optional keyword arguments are loop (to set the event loop
38 instance to use) and limit (to set the buffer limit passed to the
39 StreamReader).
40
41 (If you want to customize the StreamReader and/or
42 StreamReaderProtocol classes, just copy the code -- there's
43 really nothing special here except some convenience.)
44 """
45 loop = events.get_running_loop()
46 reader = StreamReader(limit=limit, loop=loop)
47 protocol = StreamReaderProtocol(reader, loop=loop)
48 transport, _ = await loop.create_connection(
49 lambda: protocol, host, port, **kwds)
50 writer = StreamWriter(transport, protocol, reader, loop)
51 return reader, writer
52
53
54 async def start_server(client_connected_cb, host=None, port=None, *,
55 limit=_DEFAULT_LIMIT, **kwds):
56 """Start a socket server, call back for each client connected.
57
58 The first parameter, `client_connected_cb`, takes two parameters:
59 client_reader, client_writer. client_reader is a StreamReader
60 object, while client_writer is a StreamWriter object. This
61 parameter can either be a plain callback function or a coroutine;
62 if it is a coroutine, it will be automatically converted into a
63 Task.
64
65 The rest of the arguments are all the usual arguments to
66 loop.create_server() except protocol_factory; most common are
67 positional host and port, with various optional keyword arguments
68 following. The return value is the same as loop.create_server().
69
70 Additional optional keyword argument is limit (to set the buffer
71 limit passed to the StreamReader).
72
73 The return value is the same as loop.create_server(), i.e. a
74 Server object which can be used to stop the service.
75 """
76 loop = events.get_running_loop()
77
78 def factory():
79 reader = StreamReader(limit=limit, loop=loop)
80 protocol = StreamReaderProtocol(reader, client_connected_cb,
81 loop=loop)
82 return protocol
83
84 return await loop.create_server(factory, host, port, **kwds)
85
86
87 if hasattr(socket, 'AF_UNIX'):
88 # UNIX Domain Sockets are supported on this platform
89
90 async def open_unix_connection(path=None, *,
91 limit=_DEFAULT_LIMIT, **kwds):
92 """Similar to `open_connection` but works with UNIX Domain Sockets."""
93 loop = events.get_running_loop()
94
95 reader = StreamReader(limit=limit, loop=loop)
96 protocol = StreamReaderProtocol(reader, loop=loop)
97 transport, _ = await loop.create_unix_connection(
98 lambda: protocol, path, **kwds)
99 writer = StreamWriter(transport, protocol, reader, loop)
100 return reader, writer
101
102 async def start_unix_server(client_connected_cb, path=None, *,
103 limit=_DEFAULT_LIMIT, **kwds):
104 """Similar to `start_server` but works with UNIX Domain Sockets."""
105 loop = events.get_running_loop()
106
107 def factory():
108 reader = StreamReader(limit=limit, loop=loop)
109 protocol = StreamReaderProtocol(reader, client_connected_cb,
110 loop=loop)
111 return protocol
112
113 return await loop.create_unix_server(factory, path, **kwds)
114
115
116 class FlowControlMixin(protocols.Protocol):
117 """Reusable flow control logic for StreamWriter.drain().
118
119 This implements the protocol methods pause_writing(),
120 resume_writing() and connection_lost(). If the subclass overrides
121 these it must call the super methods.
122
123 StreamWriter.drain() must wait for _drain_helper() coroutine.
124 """
125
126 def __init__(self, loop=None):
127 if loop is None:
128 self._loop = events.get_event_loop()
129 else:
130 self._loop = loop
131 self._paused = False
132 self._drain_waiters = collections.deque()
133 self._connection_lost = False
134
135 def pause_writing(self):
136 assert not self._paused
137 self._paused = True
138 if self._loop.get_debug():
139 logger.debug("%r pauses writing", self)
140
141 def resume_writing(self):
142 assert self._paused
143 self._paused = False
144 if self._loop.get_debug():
145 logger.debug("%r resumes writing", self)
146
147 for waiter in self._drain_waiters:
148 if not waiter.done():
149 waiter.set_result(None)
150
151 def connection_lost(self, exc):
152 self._connection_lost = True
153 # Wake up the writer(s) if currently paused.
154 if not self._paused:
155 return
156
157 for waiter in self._drain_waiters:
158 if not waiter.done():
159 if exc is None:
160 waiter.set_result(None)
161 else:
162 waiter.set_exception(exc)
163
164 async def _drain_helper(self):
165 if self._connection_lost:
166 raise ConnectionResetError('Connection lost')
167 if not self._paused:
168 return
169 waiter = self._loop.create_future()
170 self._drain_waiters.append(waiter)
171 try:
172 await waiter
173 finally:
174 self._drain_waiters.remove(waiter)
175
176 def _get_close_waiter(self, stream):
177 raise NotImplementedError
178
179
180 class StreamReaderProtocol(FlowControlMixin, protocols.Protocol):
181 """Helper class to adapt between Protocol and StreamReader.
182
183 (This is a helper class instead of making StreamReader itself a
184 Protocol subclass, because the StreamReader has other potential
185 uses, and to prevent the user of the StreamReader to accidentally
186 call inappropriate methods of the protocol.)
187 """
188
189 _source_traceback = None
190
191 def __init__(self, stream_reader, client_connected_cb=None, loop=None):
192 super().__init__(loop=loop)
193 if stream_reader is not None:
194 self._stream_reader_wr = weakref.ref(stream_reader)
195 self._source_traceback = stream_reader._source_traceback
196 else:
197 self._stream_reader_wr = None
198 if client_connected_cb is not None:
199 # This is a stream created by the `create_server()` function.
200 # Keep a strong reference to the reader until a connection
201 # is established.
202 self._strong_reader = stream_reader
203 self._reject_connection = False
204 self._task = None
205 self._transport = None
206 self._client_connected_cb = client_connected_cb
207 self._over_ssl = False
208 self._closed = self._loop.create_future()
209
210 @property
211 def _stream_reader(self):
212 if self._stream_reader_wr is None:
213 return None
214 return self._stream_reader_wr()
215
216 def _replace_transport(self, transport):
217 loop = self._loop
218 self._transport = transport
219 self._over_ssl = transport.get_extra_info('sslcontext') is not None
220 reader = self._stream_reader
221 if reader is not None:
222 reader._replace_transport(transport)
223
224 def connection_made(self, transport):
225 if self._reject_connection:
226 context = {
227 'message': ('An open stream was garbage collected prior to '
228 'establishing network connection; '
229 'call "stream.close()" explicitly.')
230 }
231 if self._source_traceback:
232 context['source_traceback'] = self._source_traceback
233 self._loop.call_exception_handler(context)
234 transport.abort()
235 return
236 self._transport = transport
237 reader = self._stream_reader
238 if reader is not None:
239 reader.set_transport(transport)
240 self._over_ssl = transport.get_extra_info('sslcontext') is not None
241 if self._client_connected_cb is not None:
242 writer = StreamWriter(transport, self, reader, self._loop)
243 res = self._client_connected_cb(reader, writer)
244 if coroutines.iscoroutine(res):
245 def callback(task):
246 if task.cancelled():
247 transport.close()
248 return
249 exc = task.exception()
250 if exc is not None:
251 self._loop.call_exception_handler({
252 'message': 'Unhandled exception in client_connected_cb',
253 'exception': exc,
254 'transport': transport,
255 })
256 transport.close()
257
258 self._task = self._loop.create_task(res)
259 self._task.add_done_callback(callback)
260
261 self._strong_reader = None
262
263 def connection_lost(self, exc):
264 reader = self._stream_reader
265 if reader is not None:
266 if exc is None:
267 reader.feed_eof()
268 else:
269 reader.set_exception(exc)
270 if not self._closed.done():
271 if exc is None:
272 self._closed.set_result(None)
273 else:
274 self._closed.set_exception(exc)
275 super().connection_lost(exc)
276 self._stream_reader_wr = None
277 self._stream_writer = None
278 self._task = None
279 self._transport = None
280
281 def data_received(self, data):
282 reader = self._stream_reader
283 if reader is not None:
284 reader.feed_data(data)
285
286 def eof_received(self):
287 reader = self._stream_reader
288 if reader is not None:
289 reader.feed_eof()
290 if self._over_ssl:
291 # Prevent a warning in SSLProtocol.eof_received:
292 # "returning true from eof_received()
293 # has no effect when using ssl"
294 return False
295 return True
296
297 def _get_close_waiter(self, stream):
298 return self._closed
299
300 def __del__(self):
301 # Prevent reports about unhandled exceptions.
302 # Better than self._closed._log_traceback = False hack
303 try:
304 closed = self._closed
305 except AttributeError:
306 pass # failed constructor
307 else:
308 if closed.done() and not closed.cancelled():
309 closed.exception()
310
311
312 class StreamWriter:
313 """Wraps a Transport.
314
315 This exposes write(), writelines(), [can_]write_eof(),
316 get_extra_info() and close(). It adds drain() which returns an
317 optional Future on which you can wait for flow control. It also
318 adds a transport property which references the Transport
319 directly.
320 """
321
322 def __init__(self, transport, protocol, reader, loop):
323 self._transport = transport
324 self._protocol = protocol
325 # drain() expects that the reader has an exception() method
326 assert reader is None or isinstance(reader, StreamReader)
327 self._reader = reader
328 self._loop = loop
329 self._complete_fut = self._loop.create_future()
330 self._complete_fut.set_result(None)
331
332 def __repr__(self):
333 info = [self.__class__.__name__, f'transport={self._transport!r}']
334 if self._reader is not None:
335 info.append(f'reader={self._reader!r}')
336 return '<{}>'.format(' '.join(info))
337
338 @property
339 def transport(self):
340 return self._transport
341
342 def write(self, data):
343 self._transport.write(data)
344
345 def writelines(self, data):
346 self._transport.writelines(data)
347
348 def write_eof(self):
349 return self._transport.write_eof()
350
351 def can_write_eof(self):
352 return self._transport.can_write_eof()
353
354 def close(self):
355 return self._transport.close()
356
357 def is_closing(self):
358 return self._transport.is_closing()
359
360 async def wait_closed(self):
361 await self._protocol._get_close_waiter(self)
362
363 def get_extra_info(self, name, default=None):
364 return self._transport.get_extra_info(name, default)
365
366 async def drain(self):
367 """Flush the write buffer.
368
369 The intended use is to write
370
371 w.write(data)
372 await w.drain()
373 """
374 if self._reader is not None:
375 exc = self._reader.exception()
376 if exc is not None:
377 raise exc
378 if self._transport.is_closing():
379 # Wait for protocol.connection_lost() call
380 # Raise connection closing error if any,
381 # ConnectionResetError otherwise
382 # Yield to the event loop so connection_lost() may be
383 # called. Without this, _drain_helper() would return
384 # immediately, and code that calls
385 # write(...); await drain()
386 # in a loop would never call connection_lost(), so it
387 # would not see an error when the socket is closed.
388 await sleep(0)
389 await self._protocol._drain_helper()
390
391 async def start_tls(self, sslcontext, *,
392 server_hostname=None,
393 ssl_handshake_timeout=None,
394 ssl_shutdown_timeout=None):
395 """Upgrade an existing stream-based connection to TLS."""
396 server_side = self._protocol._client_connected_cb is not None
397 protocol = self._protocol
398 await self.drain()
399 new_transport = await self._loop.start_tls( # type: ignore
400 self._transport, protocol, sslcontext,
401 server_side=server_side, server_hostname=server_hostname,
402 ssl_handshake_timeout=ssl_handshake_timeout,
403 ssl_shutdown_timeout=ssl_shutdown_timeout)
404 self._transport = new_transport
405 protocol._replace_transport(new_transport)
406
407 def __del__(self, warnings=warnings):
408 if not self._transport.is_closing():
409 if self._loop.is_closed():
410 warnings.warn("loop is closed", ResourceWarning)
411 else:
412 self.close()
413 warnings.warn(f"unclosed {self!r}", ResourceWarning)
414
415 class StreamReader:
416
417 _source_traceback = None
418
419 def __init__(self, limit=_DEFAULT_LIMIT, loop=None):
420 # The line length limit is a security feature;
421 # it also doubles as half the buffer limit.
422
423 if limit <= 0:
424 raise ValueError('Limit cannot be <= 0')
425
426 self._limit = limit
427 if loop is None:
428 self._loop = events.get_event_loop()
429 else:
430 self._loop = loop
431 self._buffer = bytearray()
432 self._eof = False # Whether we're done.
433 self._waiter = None # A future used by _wait_for_data()
434 self._exception = None
435 self._transport = None
436 self._paused = False
437 if self._loop.get_debug():
438 self._source_traceback = format_helpers.extract_stack(
439 sys._getframe(1))
440
441 def __repr__(self):
442 info = ['StreamReader']
443 if self._buffer:
444 info.append(f'{len(self._buffer)} bytes')
445 if self._eof:
446 info.append('eof')
447 if self._limit != _DEFAULT_LIMIT:
448 info.append(f'limit={self._limit}')
449 if self._waiter:
450 info.append(f'waiter={self._waiter!r}')
451 if self._exception:
452 info.append(f'exception={self._exception!r}')
453 if self._transport:
454 info.append(f'transport={self._transport!r}')
455 if self._paused:
456 info.append('paused')
457 return '<{}>'.format(' '.join(info))
458
459 def exception(self):
460 return self._exception
461
462 def set_exception(self, exc):
463 self._exception = exc
464
465 waiter = self._waiter
466 if waiter is not None:
467 self._waiter = None
468 if not waiter.cancelled():
469 waiter.set_exception(exc)
470
471 def _wakeup_waiter(self):
472 """Wakeup read*() functions waiting for data or EOF."""
473 waiter = self._waiter
474 if waiter is not None:
475 self._waiter = None
476 if not waiter.cancelled():
477 waiter.set_result(None)
478
479 def set_transport(self, transport):
480 assert self._transport is None, 'Transport already set'
481 self._transport = transport
482
483 def _replace_transport(self, transport):
484 assert self._transport is not None, 'Transport not set'
485 self._transport = transport
486
487 def _maybe_resume_transport(self):
488 if self._paused and len(self._buffer) <= self._limit:
489 self._paused = False
490 self._transport.resume_reading()
491
492 def feed_eof(self):
493 self._eof = True
494 self._wakeup_waiter()
495
496 def at_eof(self):
497 """Return True if the buffer is empty and 'feed_eof' was called."""
498 return self._eof and not self._buffer
499
500 def feed_data(self, data):
501 assert not self._eof, 'feed_data after feed_eof'
502
503 if not data:
504 return
505
506 self._buffer.extend(data)
507 self._wakeup_waiter()
508
509 if (self._transport is not None and
510 not self._paused and
511 len(self._buffer) > 2 * self._limit):
512 try:
513 self._transport.pause_reading()
514 except NotImplementedError:
515 # The transport can't be paused.
516 # We'll just have to buffer all data.
517 # Forget the transport so we don't keep trying.
518 self._transport = None
519 else:
520 self._paused = True
521
522 async def _wait_for_data(self, func_name):
523 """Wait until feed_data() or feed_eof() is called.
524
525 If stream was paused, automatically resume it.
526 """
527 # StreamReader uses a future to link the protocol feed_data() method
528 # to a read coroutine. Running two read coroutines at the same time
529 # would have an unexpected behaviour. It would not possible to know
530 # which coroutine would get the next data.
531 if self._waiter is not None:
532 raise RuntimeError(
533 f'{func_name}() called while another coroutine is '
534 f'already waiting for incoming data')
535
536 assert not self._eof, '_wait_for_data after EOF'
537
538 # Waiting for data while paused will make deadlock, so prevent it.
539 # This is essential for readexactly(n) for case when n > self._limit.
540 if self._paused:
541 self._paused = False
542 self._transport.resume_reading()
543
544 self._waiter = self._loop.create_future()
545 try:
546 await self._waiter
547 finally:
548 self._waiter = None
549
550 async def readline(self):
551 r"""Read chunk of data from the stream until newline (b'\n') is found.
552
553 On success, return chunk that ends with newline. If only partial
554 line can be read due to EOF, return incomplete line without
555 terminating newline. When EOF was reached while no bytes read,
556 empty bytes object is returned.
557
558 If limit is reached, ValueError will be raised. In that case, if
559 newline was found, complete line including newline will be removed
560 from internal buffer. Else, internal buffer will be cleared.
561 Limit is compared against part of the line without newline.
562
563 If stream was paused, this function will automatically resume it if
564 needed.
565 """
566 sep = b'\n'
567 seplen = len(sep)
568 try:
569 line = await self.readuntil(sep)
570 except exceptions.IncompleteReadError as e:
571 return e.partial
572 except exceptions.LimitOverrunError as e:
573 if self._buffer.startswith(sep, e.consumed):
574 del self._buffer[:e.consumed + seplen]
575 else:
576 self._buffer.clear()
577 self._maybe_resume_transport()
578 raise ValueError(e.args[0])
579 return line
580
581 async def readuntil(self, separator=b'\n'):
582 """Read data from the stream until ``separator`` is found.
583
584 On success, the data and separator will be removed from the
585 internal buffer (consumed). Returned data will include the
586 separator at the end.
587
588 Configured stream limit is used to check result. Limit sets the
589 maximal length of data that can be returned, not counting the
590 separator.
591
592 If an EOF occurs and the complete separator is still not found,
593 an IncompleteReadError exception will be raised, and the internal
594 buffer will be reset. The IncompleteReadError.partial attribute
595 may contain the separator partially.
596
597 If the data cannot be read because of over limit, a
598 LimitOverrunError exception will be raised, and the data
599 will be left in the internal buffer, so it can be read again.
600
601 The ``separator`` may also be a tuple of separators. In this
602 case the return value will be the shortest possible that has any
603 separator as the suffix. For the purposes of LimitOverrunError,
604 the shortest possible separator is considered to be the one that
605 matched.
606 """
607 if isinstance(separator, tuple):
608 # Makes sure shortest matches wins
609 separator = sorted(separator, key=len)
610 else:
611 separator = [separator]
612 if not separator:
613 raise ValueError('Separator should contain at least one element')
614 min_seplen = len(separator[0])
615 max_seplen = len(separator[-1])
616 if min_seplen == 0:
617 raise ValueError('Separator should be at least one-byte string')
618
619 if self._exception is not None:
620 raise self._exception
621
622 # Consume whole buffer except last bytes, which length is
623 # one less than max_seplen. Let's check corner cases with
624 # separator[-1]='SEPARATOR':
625 # * we have received almost complete separator (without last
626 # byte). i.e buffer='some textSEPARATO'. In this case we
627 # can safely consume max_seplen - 1 bytes.
628 # * last byte of buffer is first byte of separator, i.e.
629 # buffer='abcdefghijklmnopqrS'. We may safely consume
630 # everything except that last byte, but this require to
631 # analyze bytes of buffer that match partial separator.
632 # This is slow and/or require FSM. For this case our
633 # implementation is not optimal, since require rescanning
634 # of data that is known to not belong to separator. In
635 # real world, separator will not be so long to notice
636 # performance problems. Even when reading MIME-encoded
637 # messages :)
638
639 # `offset` is the number of bytes from the beginning of the buffer
640 # where there is no occurrence of any `separator`.
641 offset = 0
642
643 # Loop until we find a `separator` in the buffer, exceed the buffer size,
644 # or an EOF has happened.
645 while True:
646 buflen = len(self._buffer)
647
648 # Check if we now have enough data in the buffer for shortest
649 # separator to fit.
650 if buflen - offset >= min_seplen:
651 match_start = None
652 match_end = None
653 for sep in separator:
654 isep = self._buffer.find(sep, offset)
655
656 if isep != -1:
657 # `separator` is in the buffer. `match_start` and
658 # `match_end` will be used later to retrieve the
659 # data.
660 end = isep + len(sep)
661 if match_end is None or end < match_end:
662 match_end = end
663 match_start = isep
664 if match_end is not None:
665 break
666
667 # see upper comment for explanation.
668 offset = max(0, buflen + 1 - max_seplen)
669 if offset > self._limit:
670 raise exceptions.LimitOverrunError(
671 'Separator is not found, and chunk exceed the limit',
672 offset)
673
674 # Complete message (with full separator) may be present in buffer
675 # even when EOF flag is set. This may happen when the last chunk
676 # adds data which makes separator be found. That's why we check for
677 # EOF *after* inspecting the buffer.
678 if self._eof:
679 chunk = bytes(self._buffer)
680 self._buffer.clear()
681 raise exceptions.IncompleteReadError(chunk, None)
682
683 # _wait_for_data() will resume reading if stream was paused.
684 await self._wait_for_data('readuntil')
685
686 if match_start > self._limit:
687 raise exceptions.LimitOverrunError(
688 'Separator is found, but chunk is longer than limit', match_start)
689
690 chunk = self._buffer[:match_end]
691 del self._buffer[:match_end]
692 self._maybe_resume_transport()
693 return bytes(chunk)
694
695 async def read(self, n=-1):
696 """Read up to `n` bytes from the stream.
697
698 If `n` is not provided or set to -1,
699 read until EOF, then return all read bytes.
700 If EOF was received and the internal buffer is empty,
701 return an empty bytes object.
702
703 If `n` is 0, return an empty bytes object immediately.
704
705 If `n` is positive, return at most `n` available bytes
706 as soon as at least 1 byte is available in the internal buffer.
707 If EOF is received before any byte is read, return an empty
708 bytes object.
709
710 Returned value is not limited with limit, configured at stream
711 creation.
712
713 If stream was paused, this function will automatically resume it if
714 needed.
715 """
716
717 if self._exception is not None:
718 raise self._exception
719
720 if n == 0:
721 return b''
722
723 if n < 0:
724 # This used to just loop creating a new waiter hoping to
725 # collect everything in self._buffer, but that would
726 # deadlock if the subprocess sends more than self.limit
727 # bytes. So just call self.read(self._limit) until EOF.
728 blocks = []
729 while True:
730 block = await self.read(self._limit)
731 if not block:
732 break
733 blocks.append(block)
734 return b''.join(blocks)
735
736 if not self._buffer and not self._eof:
737 await self._wait_for_data('read')
738
739 # This will work right even if buffer is less than n bytes
740 data = bytes(memoryview(self._buffer)[:n])
741 del self._buffer[:n]
742
743 self._maybe_resume_transport()
744 return data
745
746 async def readexactly(self, n):
747 """Read exactly `n` bytes.
748
749 Raise an IncompleteReadError if EOF is reached before `n` bytes can be
750 read. The IncompleteReadError.partial attribute of the exception will
751 contain the partial read bytes.
752
753 if n is zero, return empty bytes object.
754
755 Returned value is not limited with limit, configured at stream
756 creation.
757
758 If stream was paused, this function will automatically resume it if
759 needed.
760 """
761 if n < 0:
762 raise ValueError('readexactly size can not be less than zero')
763
764 if self._exception is not None:
765 raise self._exception
766
767 if n == 0:
768 return b''
769
770 while len(self._buffer) < n:
771 if self._eof:
772 incomplete = bytes(self._buffer)
773 self._buffer.clear()
774 raise exceptions.IncompleteReadError(incomplete, n)
775
776 await self._wait_for_data('readexactly')
777
778 if len(self._buffer) == n:
779 data = bytes(self._buffer)
780 self._buffer.clear()
781 else:
782 data = bytes(memoryview(self._buffer)[:n])
783 del self._buffer[:n]
784 self._maybe_resume_transport()
785 return data
786
787 def __aiter__(self):
788 return self
789
790 async def __anext__(self):
791 val = await self.readline()
792 if val == b'':
793 raise StopAsyncIteration
794 return val