Debian Codegraph @

azure-kusto-ingest/azure/kusto/ingest/_stream_extensions.py

raw
1 import io
2
3 from typing import IO, AnyStr
4
5
6 def read_until_size_or_end(stream: IO[AnyStr], size: int) -> io.BytesIO:
7 pos = 0
8 result = io.BytesIO()
9 while True:
10 try:
11 returned = stream.read(size - pos)
12 pos += len(returned)
13 result.write(returned)
14
15 if len(returned) == 0 or pos == size:
16 result.seek(0, io.SEEK_SET)
17 return result
18
19 except BlockingIOError:
20 continue
21
22
23 class ChainStream(io.RawIOBase):
24 """
25 https://stackoverflow.com/questions/24528278/stream-multiple-files-into-a-readable-object-in-python
26 """
27
28 def __init__(self, streams):
29 self.leftover = b""
30 self.stream_iter = iter(streams)
31 try:
32 self.stream = next(self.stream_iter)
33 except StopIteration:
34 self.stream = None
35
36 def readable(self):
37 return True
38
39 def _read_next_chunk(self, max_length):
40 # Return 0 or more bytes from the current stream, first returning all
41 # leftover bytes. If the stream is closed returns b''
42 if self.leftover:
43 return self.leftover
44 elif self.stream is not None:
45 return self.stream.read(max_length)
46 else:
47 return b""
48
49 def readinto(self, b):
50 buffer_length = len(b)
51 chunk = self._read_next_chunk(buffer_length)
52 while len(chunk) == 0:
53 # move to next stream
54 if self.stream is not None:
55 self.stream.close()
56 try:
57 self.stream = next(self.stream_iter)
58 chunk = self._read_next_chunk(buffer_length)
59 except StopIteration:
60 # No more streams to chain together
61 self.stream = None
62 return 0 # indicate EOF
63 output, self.leftover = chunk[:buffer_length], chunk[buffer_length:]
64 b[: len(output)] = output
65 return len(output)
66
67
68 def chain_streams(streams, buffer_size=io.DEFAULT_BUFFER_SIZE):
69 """
70 Chain an iterable of streams together into a single buffered stream.
71 Usage:
72 def generate_open_file_streams():
73 for file in filenames:
74 yield open(file, 'rb')
75 f = chain_streams(generate_open_file_streams())
76 f.read()
77 """
78 return io.BufferedReader(ChainStream(streams), buffer_size=buffer_size)