Debian Codegraph @

Lib/test/test_io.py

raw
1 """Unit tests for the io module."""
2
3 # Tests of io are scattered over the test suite:
4 # * test_bufio - tests file buffering
5 # * test_memoryio - tests BytesIO and StringIO
6 # * test_fileio - tests FileIO
7 # * test_file - tests the file interface
8 # * test_io - tests everything else in the io module
9 # * test_univnewlines - tests universal newline support
10 # * test_largefile - tests operations on a file greater than 2**32 bytes
11 # (only enabled with -ulargefile)
12
13 ################################################################################
14 # ATTENTION TEST WRITERS!!!
15 ################################################################################
16 # When writing tests for io, it's important to test both the C and Python
17 # implementations. This is usually done by writing a base test that refers to
18 # the type it is testing as an attribute. Then it provides custom subclasses to
19 # test both implementations. This file has lots of examples.
20 ################################################################################
21
22 import abc
23 import array
24 import errno
25 import locale
26 import os
27 import pickle
28 import random
29 import signal
30 import sys
31 import textwrap
32 import threading
33 import time
34 import unittest
35 import warnings
36 import weakref
37 from collections import deque, UserList
38 from itertools import cycle, count
39 from test import support
40 from test.support.script_helper import (
41 assert_python_ok, assert_python_failure, run_python_until_end)
42 from test.support import (
43 import_helper, is_apple, os_helper, threading_helper, warnings_helper,
44 )
45 from test.support.os_helper import FakePath
46
47 import codecs
48 import io # C implementation of io
49 import _pyio as pyio # Python implementation of io
50
51 try:
52 import ctypes
53 except ImportError:
54 def byteslike(*pos, **kw):
55 return array.array("b", bytes(*pos, **kw))
56 else:
57 def byteslike(*pos, **kw):
58 """Create a bytes-like object having no string or sequence methods"""
59 data = bytes(*pos, **kw)
60 obj = EmptyStruct()
61 ctypes.resize(obj, len(data))
62 memoryview(obj).cast("B")[:] = data
63 return obj
64 class EmptyStruct(ctypes.Structure):
65 pass
66
67
68 def _default_chunk_size():
69 """Get the default TextIOWrapper chunk size"""
70 with open(__file__, "r", encoding="latin-1") as f:
71 return f._CHUNK_SIZE
72
73 requires_alarm = unittest.skipUnless(
74 hasattr(signal, "alarm"), "test requires signal.alarm()"
75 )
76
77
78 class BadIndex:
79 def __index__(self):
80 1/0
81
82 class MockRawIOWithoutRead:
83 """A RawIO implementation without read(), so as to exercise the default
84 RawIO.read() which calls readinto()."""
85
86 def __init__(self, read_stack=()):
87 self._read_stack = list(read_stack)
88 self._write_stack = []
89 self._reads = 0
90 self._extraneous_reads = 0
91
92 def write(self, b):
93 self._write_stack.append(bytes(b))
94 return len(b)
95
96 def writable(self):
97 return True
98
99 def fileno(self):
100 return 42
101
102 def readable(self):
103 return True
104
105 def seekable(self):
106 return True
107
108 def seek(self, pos, whence):
109 return 0 # wrong but we gotta return something
110
111 def tell(self):
112 return 0 # same comment as above
113
114 def readinto(self, buf):
115 self._reads += 1
116 max_len = len(buf)
117 try:
118 data = self._read_stack[0]
119 except IndexError:
120 self._extraneous_reads += 1
121 return 0
122 if data is None:
123 del self._read_stack[0]
124 return None
125 n = len(data)
126 if len(data) <= max_len:
127 del self._read_stack[0]
128 buf[:n] = data
129 return n
130 else:
131 buf[:] = data[:max_len]
132 self._read_stack[0] = data[max_len:]
133 return max_len
134
135 def truncate(self, pos=None):
136 return pos
137
138 class CMockRawIOWithoutRead(MockRawIOWithoutRead, io.RawIOBase):
139 pass
140
141 class PyMockRawIOWithoutRead(MockRawIOWithoutRead, pyio.RawIOBase):
142 pass
143
144
145 class MockRawIO(MockRawIOWithoutRead):
146
147 def read(self, n=None):
148 self._reads += 1
149 try:
150 return self._read_stack.pop(0)
151 except:
152 self._extraneous_reads += 1
153 return b""
154
155 class CMockRawIO(MockRawIO, io.RawIOBase):
156 pass
157
158 class PyMockRawIO(MockRawIO, pyio.RawIOBase):
159 pass
160
161
162 class MisbehavedRawIO(MockRawIO):
163 def write(self, b):
164 return super().write(b) * 2
165
166 def read(self, n=None):
167 return super().read(n) * 2
168
169 def seek(self, pos, whence):
170 return -123
171
172 def tell(self):
173 return -456
174
175 def readinto(self, buf):
176 super().readinto(buf)
177 return len(buf) * 5
178
179 class CMisbehavedRawIO(MisbehavedRawIO, io.RawIOBase):
180 pass
181
182 class PyMisbehavedRawIO(MisbehavedRawIO, pyio.RawIOBase):
183 pass
184
185
186 class SlowFlushRawIO(MockRawIO):
187 def __init__(self):
188 super().__init__()
189 self.in_flush = threading.Event()
190
191 def flush(self):
192 self.in_flush.set()
193 time.sleep(0.25)
194
195 class CSlowFlushRawIO(SlowFlushRawIO, io.RawIOBase):
196 pass
197
198 class PySlowFlushRawIO(SlowFlushRawIO, pyio.RawIOBase):
199 pass
200
201
202 class CloseFailureIO(MockRawIO):
203 closed = 0
204
205 def close(self):
206 if not self.closed:
207 self.closed = 1
208 raise OSError
209
210 class CCloseFailureIO(CloseFailureIO, io.RawIOBase):
211 pass
212
213 class PyCloseFailureIO(CloseFailureIO, pyio.RawIOBase):
214 pass
215
216
217 class MockFileIO:
218
219 def __init__(self, data):
220 self.read_history = []
221 super().__init__(data)
222
223 def read(self, n=None):
224 res = super().read(n)
225 self.read_history.append(None if res is None else len(res))
226 return res
227
228 def readinto(self, b):
229 res = super().readinto(b)
230 self.read_history.append(res)
231 return res
232
233 class CMockFileIO(MockFileIO, io.BytesIO):
234 pass
235
236 class PyMockFileIO(MockFileIO, pyio.BytesIO):
237 pass
238
239
240 class MockUnseekableIO:
241 def seekable(self):
242 return False
243
244 def seek(self, *args):
245 raise self.UnsupportedOperation("not seekable")
246
247 def tell(self, *args):
248 raise self.UnsupportedOperation("not seekable")
249
250 def truncate(self, *args):
251 raise self.UnsupportedOperation("not seekable")
252
253 class CMockUnseekableIO(MockUnseekableIO, io.BytesIO):
254 UnsupportedOperation = io.UnsupportedOperation
255
256 class PyMockUnseekableIO(MockUnseekableIO, pyio.BytesIO):
257 UnsupportedOperation = pyio.UnsupportedOperation
258
259
260 class MockCharPseudoDevFileIO(MockFileIO):
261 # GH-95782
262 # ftruncate() does not work on these special files (and CPython then raises
263 # appropriate exceptions), so truncate() does not have to be accounted for
264 # here.
265 def __init__(self, data):
266 super().__init__(data)
267
268 def seek(self, *args):
269 return 0
270
271 def tell(self, *args):
272 return 0
273
274 class CMockCharPseudoDevFileIO(MockCharPseudoDevFileIO, io.BytesIO):
275 pass
276
277 class PyMockCharPseudoDevFileIO(MockCharPseudoDevFileIO, pyio.BytesIO):
278 pass
279
280
281 class MockNonBlockWriterIO:
282
283 def __init__(self):
284 self._write_stack = []
285 self._blocker_char = None
286
287 def pop_written(self):
288 s = b"".join(self._write_stack)
289 self._write_stack[:] = []
290 return s
291
292 def block_on(self, char):
293 """Block when a given char is encountered."""
294 self._blocker_char = char
295
296 def readable(self):
297 return True
298
299 def seekable(self):
300 return True
301
302 def seek(self, pos, whence=0):
303 # naive implementation, enough for tests
304 return 0
305
306 def writable(self):
307 return True
308
309 def write(self, b):
310 b = bytes(b)
311 n = -1
312 if self._blocker_char:
313 try:
314 n = b.index(self._blocker_char)
315 except ValueError:
316 pass
317 else:
318 if n > 0:
319 # write data up to the first blocker
320 self._write_stack.append(b[:n])
321 return n
322 else:
323 # cancel blocker and indicate would block
324 self._blocker_char = None
325 return None
326 self._write_stack.append(b)
327 return len(b)
328
329 class CMockNonBlockWriterIO(MockNonBlockWriterIO, io.RawIOBase):
330 BlockingIOError = io.BlockingIOError
331
332 class PyMockNonBlockWriterIO(MockNonBlockWriterIO, pyio.RawIOBase):
333 BlockingIOError = pyio.BlockingIOError
334
335
336 class IOTest(unittest.TestCase):
337
338 def setUp(self):
339 os_helper.unlink(os_helper.TESTFN)
340
341 def tearDown(self):
342 os_helper.unlink(os_helper.TESTFN)
343
344 def write_ops(self, f):
345 self.assertEqual(f.write(b"blah."), 5)
346 f.truncate(0)
347 self.assertEqual(f.tell(), 5)
348 f.seek(0)
349
350 self.assertEqual(f.write(b"blah."), 5)
351 self.assertEqual(f.seek(0), 0)
352 self.assertEqual(f.write(b"Hello."), 6)
353 self.assertEqual(f.tell(), 6)
354 self.assertEqual(f.seek(-1, 1), 5)
355 self.assertEqual(f.tell(), 5)
356 buffer = bytearray(b" world\n\n\n")
357 self.assertEqual(f.write(buffer), 9)
358 buffer[:] = b"*" * 9 # Overwrite our copy of the data
359 self.assertEqual(f.seek(0), 0)
360 self.assertEqual(f.write(b"h"), 1)
361 self.assertEqual(f.seek(-1, 2), 13)
362 self.assertEqual(f.tell(), 13)
363
364 self.assertEqual(f.truncate(12), 12)
365 self.assertEqual(f.tell(), 13)
366 self.assertRaises(TypeError, f.seek, 0.0)
367
368 def read_ops(self, f, buffered=False):
369 data = f.read(5)
370 self.assertEqual(data, b"hello")
371 data = byteslike(data)
372 self.assertEqual(f.readinto(data), 5)
373 self.assertEqual(bytes(data), b" worl")
374 data = bytearray(5)
375 self.assertEqual(f.readinto(data), 2)
376 self.assertEqual(len(data), 5)
377 self.assertEqual(data[:2], b"d\n")
378 self.assertEqual(f.seek(0), 0)
379 self.assertEqual(f.read(20), b"hello world\n")
380 self.assertEqual(f.read(1), b"")
381 self.assertEqual(f.readinto(byteslike(b"x")), 0)
382 self.assertEqual(f.seek(-6, 2), 6)
383 self.assertEqual(f.read(5), b"world")
384 self.assertEqual(f.read(0), b"")
385 self.assertEqual(f.readinto(byteslike()), 0)
386 self.assertEqual(f.seek(-6, 1), 5)
387 self.assertEqual(f.read(5), b" worl")
388 self.assertEqual(f.tell(), 10)
389 self.assertRaises(TypeError, f.seek, 0.0)
390 if buffered:
391 f.seek(0)
392 self.assertEqual(f.read(), b"hello world\n")
393 f.seek(6)
394 self.assertEqual(f.read(), b"world\n")
395 self.assertEqual(f.read(), b"")
396 f.seek(0)
397 data = byteslike(5)
398 self.assertEqual(f.readinto1(data), 5)
399 self.assertEqual(bytes(data), b"hello")
400
401 LARGE = 2**31
402
403 def large_file_ops(self, f):
404 assert f.readable()
405 assert f.writable()
406 try:
407 self.assertEqual(f.seek(self.LARGE), self.LARGE)
408 except (OverflowError, ValueError):
409 self.skipTest("no largefile support")
410 self.assertEqual(f.tell(), self.LARGE)
411 self.assertEqual(f.write(b"xxx"), 3)
412 self.assertEqual(f.tell(), self.LARGE + 3)
413 self.assertEqual(f.seek(-1, 1), self.LARGE + 2)
414 self.assertEqual(f.truncate(), self.LARGE + 2)
415 self.assertEqual(f.tell(), self.LARGE + 2)
416 self.assertEqual(f.seek(0, 2), self.LARGE + 2)
417 self.assertEqual(f.truncate(self.LARGE + 1), self.LARGE + 1)
418 self.assertEqual(f.tell(), self.LARGE + 2)
419 self.assertEqual(f.seek(0, 2), self.LARGE + 1)
420 self.assertEqual(f.seek(-1, 2), self.LARGE)
421 self.assertEqual(f.read(2), b"x")
422
423 def test_invalid_operations(self):
424 # Try writing on a file opened in read mode and vice-versa.
425 exc = self.UnsupportedOperation
426 with self.open(os_helper.TESTFN, "w", encoding="utf-8") as fp:
427 self.assertRaises(exc, fp.read)
428 self.assertRaises(exc, fp.readline)
429 with self.open(os_helper.TESTFN, "wb") as fp:
430 self.assertRaises(exc, fp.read)
431 self.assertRaises(exc, fp.readline)
432 with self.open(os_helper.TESTFN, "wb", buffering=0) as fp:
433 self.assertRaises(exc, fp.read)
434 self.assertRaises(exc, fp.readline)
435 with self.open(os_helper.TESTFN, "rb", buffering=0) as fp:
436 self.assertRaises(exc, fp.write, b"blah")
437 self.assertRaises(exc, fp.writelines, [b"blah\n"])
438 with self.open(os_helper.TESTFN, "rb") as fp:
439 self.assertRaises(exc, fp.write, b"blah")
440 self.assertRaises(exc, fp.writelines, [b"blah\n"])
441 with self.open(os_helper.TESTFN, "r", encoding="utf-8") as fp:
442 self.assertRaises(exc, fp.write, "blah")
443 self.assertRaises(exc, fp.writelines, ["blah\n"])
444 # Non-zero seeking from current or end pos
445 self.assertRaises(exc, fp.seek, 1, self.SEEK_CUR)
446 self.assertRaises(exc, fp.seek, -1, self.SEEK_END)
447
448 @support.cpython_only
449 def test_startup_optimization(self):
450 # gh-132952: Test that `io` is not imported at startup and that the
451 # __module__ of UnsupportedOperation is set to "io".
452 assert_python_ok("-S", "-c", textwrap.dedent(
453 """
454 import sys
455 assert "io" not in sys.modules
456 try:
457 sys.stdin.truncate()
458 except Exception as e:
459 typ = type(e)
460 assert typ.__module__ == "io", (typ, typ.__module__)
461 assert typ.__name__ == "UnsupportedOperation", (typ, typ.__name__)
462 else:
463 raise AssertionError("Expected UnsupportedOperation")
464 """
465 ))
466
467 @unittest.skipUnless(hasattr(os, "pipe"), "requires os.pipe()")
468 def test_optional_abilities(self):
469 # Test for OSError when optional APIs are not supported
470 # The purpose of this test is to try fileno(), reading, writing and
471 # seeking operations with various objects that indicate they do not
472 # support these operations.
473
474 def pipe_reader():
475 [r, w] = os.pipe()
476 os.close(w) # So that read() is harmless
477 return self.FileIO(r, "r")
478
479 def pipe_writer():
480 [r, w] = os.pipe()
481 self.addCleanup(os.close, r)
482 # Guarantee that we can write into the pipe without blocking
483 thread = threading.Thread(target=os.read, args=(r, 100))
484 thread.start()
485 self.addCleanup(thread.join)
486 return self.FileIO(w, "w")
487
488 def buffered_reader():
489 return self.BufferedReader(self.MockUnseekableIO())
490
491 def buffered_writer():
492 return self.BufferedWriter(self.MockUnseekableIO())
493
494 def buffered_random():
495 return self.BufferedRandom(self.BytesIO())
496
497 def buffered_rw_pair():
498 return self.BufferedRWPair(self.MockUnseekableIO(),
499 self.MockUnseekableIO())
500
501 def text_reader():
502 class UnseekableReader(self.MockUnseekableIO):
503 writable = self.BufferedIOBase.writable
504 write = self.BufferedIOBase.write
505 return self.TextIOWrapper(UnseekableReader(), "ascii")
506
507 def text_writer():
508 class UnseekableWriter(self.MockUnseekableIO):
509 readable = self.BufferedIOBase.readable
510 read = self.BufferedIOBase.read
511 return self.TextIOWrapper(UnseekableWriter(), "ascii")
512
513 tests = (
514 (pipe_reader, "fr"), (pipe_writer, "fw"),
515 (buffered_reader, "r"), (buffered_writer, "w"),
516 (buffered_random, "rws"), (buffered_rw_pair, "rw"),
517 (text_reader, "r"), (text_writer, "w"),
518 (self.BytesIO, "rws"), (self.StringIO, "rws"),
519 )
520
521 def do_test(test, obj, abilities):
522 readable = "r" in abilities
523 self.assertEqual(obj.readable(), readable)
524 writable = "w" in abilities
525 self.assertEqual(obj.writable(), writable)
526
527 if isinstance(obj, self.TextIOBase):
528 data = "3"
529 elif isinstance(obj, (self.BufferedIOBase, self.RawIOBase)):
530 data = b"3"
531 else:
532 self.fail("Unknown base class")
533
534 if "f" in abilities:
535 obj.fileno()
536 else:
537 self.assertRaises(OSError, obj.fileno)
538
539 if readable:
540 obj.read(1)
541 obj.read()
542 else:
543 self.assertRaises(OSError, obj.read, 1)
544 self.assertRaises(OSError, obj.read)
545
546 if writable:
547 obj.write(data)
548 else:
549 self.assertRaises(OSError, obj.write, data)
550
551 if sys.platform.startswith("win") and test in (
552 pipe_reader, pipe_writer):
553 # Pipes seem to appear as seekable on Windows
554 return
555 seekable = "s" in abilities
556 self.assertEqual(obj.seekable(), seekable)
557
558 if seekable:
559 obj.tell()
560 obj.seek(0)
561 else:
562 self.assertRaises(OSError, obj.tell)
563 self.assertRaises(OSError, obj.seek, 0)
564
565 if writable and seekable:
566 obj.truncate()
567 obj.truncate(0)
568 else:
569 self.assertRaises(OSError, obj.truncate)
570 self.assertRaises(OSError, obj.truncate, 0)
571
572 for [test, abilities] in tests:
573 with self.subTest(test):
574 if test == pipe_writer and not threading_helper.can_start_thread:
575 self.skipTest("Need threads")
576 with test() as obj:
577 do_test(test, obj, abilities)
578
579
580 def test_open_handles_NUL_chars(self):
581 fn_with_NUL = 'foo\0bar'
582 self.assertRaises(ValueError, self.open, fn_with_NUL, 'w', encoding="utf-8")
583
584 bytes_fn = bytes(fn_with_NUL, 'ascii')
585 with warnings.catch_warnings():
586 warnings.simplefilter("ignore", DeprecationWarning)
587 self.assertRaises(ValueError, self.open, bytes_fn, 'w', encoding="utf-8")
588
589 def test_raw_file_io(self):
590 with self.open(os_helper.TESTFN, "wb", buffering=0) as f:
591 self.assertEqual(f.readable(), False)
592 self.assertEqual(f.writable(), True)
593 self.assertEqual(f.seekable(), True)
594 self.write_ops(f)
595 with self.open(os_helper.TESTFN, "rb", buffering=0) as f:
596 self.assertEqual(f.readable(), True)
597 self.assertEqual(f.writable(), False)
598 self.assertEqual(f.seekable(), True)
599 self.read_ops(f)
600
601 def test_buffered_file_io(self):
602 with self.open(os_helper.TESTFN, "wb") as f:
603 self.assertEqual(f.readable(), False)
604 self.assertEqual(f.writable(), True)
605 self.assertEqual(f.seekable(), True)
606 self.write_ops(f)
607 with self.open(os_helper.TESTFN, "rb") as f:
608 self.assertEqual(f.readable(), True)
609 self.assertEqual(f.writable(), False)
610 self.assertEqual(f.seekable(), True)
611 self.read_ops(f, True)
612
613 def test_readline(self):
614 with self.open(os_helper.TESTFN, "wb") as f:
615 f.write(b"abc\ndef\nxyzzy\nfoo\x00bar\nanother line")
616 with self.open(os_helper.TESTFN, "rb") as f:
617 self.assertEqual(f.readline(), b"abc\n")
618 self.assertEqual(f.readline(10), b"def\n")
619 self.assertEqual(f.readline(2), b"xy")
620 self.assertEqual(f.readline(4), b"zzy\n")
621 self.assertEqual(f.readline(), b"foo\x00bar\n")
622 self.assertEqual(f.readline(None), b"another line")
623 self.assertRaises(TypeError, f.readline, 5.3)
624 with self.open(os_helper.TESTFN, "r", encoding="utf-8") as f:
625 self.assertRaises(TypeError, f.readline, 5.3)
626
627 def test_readline_nonsizeable(self):
628 # Issue #30061
629 # Crash when readline() returns an object without __len__
630 class R(self.IOBase):
631 def readline(self):
632 return None
633 self.assertRaises((TypeError, StopIteration), next, R())
634
635 def test_next_nonsizeable(self):
636 # Issue #30061
637 # Crash when __next__() returns an object without __len__
638 class R(self.IOBase):
639 def __next__(self):
640 return None
641 self.assertRaises(TypeError, R().readlines, 1)
642
643 def test_raw_bytes_io(self):
644 f = self.BytesIO()
645 self.write_ops(f)
646 data = f.getvalue()
647 self.assertEqual(data, b"hello world\n")
648 f = self.BytesIO(data)
649 self.read_ops(f, True)
650
651 def test_large_file_ops(self):
652 # On Windows and Apple platforms this test consumes large resources; It
653 # takes a long time to build the >2 GiB file and takes >2 GiB of disk
654 # space therefore the resource must be enabled to run this test.
655 if sys.platform[:3] == 'win' or is_apple:
656 support.requires(
657 'largefile',
658 'test requires %s bytes and a long time to run' % self.LARGE)
659 with self.open(os_helper.TESTFN, "w+b", 0) as f:
660 self.large_file_ops(f)
661 with self.open(os_helper.TESTFN, "w+b") as f:
662 self.large_file_ops(f)
663
664 def test_with_open(self):
665 for bufsize in (0, 100):
666 with self.open(os_helper.TESTFN, "wb", bufsize) as f:
667 f.write(b"xxx")
668 self.assertEqual(f.closed, True)
669 try:
670 with self.open(os_helper.TESTFN, "wb", bufsize) as f:
671 1/0
672 except ZeroDivisionError:
673 self.assertEqual(f.closed, True)
674 else:
675 self.fail("1/0 didn't raise an exception")
676
677 # issue 5008
678 def test_append_mode_tell(self):
679 with self.open(os_helper.TESTFN, "wb") as f:
680 f.write(b"xxx")
681 with self.open(os_helper.TESTFN, "ab", buffering=0) as f:
682 self.assertEqual(f.tell(), 3)
683 with self.open(os_helper.TESTFN, "ab") as f:
684 self.assertEqual(f.tell(), 3)
685 with self.open(os_helper.TESTFN, "a", encoding="utf-8") as f:
686 self.assertGreater(f.tell(), 0)
687
688 def test_destructor(self):
689 record = []
690 class MyFileIO(self.FileIO):
691 def __del__(self):
692 record.append(1)
693 try:
694 f = super().__del__
695 except AttributeError:
696 pass
697 else:
698 f()
699 def close(self):
700 record.append(2)
701 super().close()
702 def flush(self):
703 record.append(3)
704 super().flush()
705 with warnings_helper.check_warnings(('', ResourceWarning)):
706 f = MyFileIO(os_helper.TESTFN, "wb")
707 f.write(b"xxx")
708 del f
709 support.gc_collect()
710 self.assertEqual(record, [1, 2, 3])
711 with self.open(os_helper.TESTFN, "rb") as f:
712 self.assertEqual(f.read(), b"xxx")
713
714 def _check_base_destructor(self, base):
715 record = []
716 class MyIO(base):
717 def __init__(self):
718 # This exercises the availability of attributes on object
719 # destruction.
720 # (in the C version, close() is called by the tp_dealloc
721 # function, not by __del__)
722 self.on_del = 1
723 self.on_close = 2
724 self.on_flush = 3
725 def __del__(self):
726 record.append(self.on_del)
727 try:
728 f = super().__del__
729 except AttributeError:
730 pass
731 else:
732 f()
733 def close(self):
734 record.append(self.on_close)
735 super().close()
736 def flush(self):
737 record.append(self.on_flush)
738 super().flush()
739 f = MyIO()
740 del f
741 support.gc_collect()
742 self.assertEqual(record, [1, 2, 3])
743
744 def test_IOBase_destructor(self):
745 self._check_base_destructor(self.IOBase)
746
747 def test_RawIOBase_destructor(self):
748 self._check_base_destructor(self.RawIOBase)
749
750 def test_BufferedIOBase_destructor(self):
751 self._check_base_destructor(self.BufferedIOBase)
752
753 def test_TextIOBase_destructor(self):
754 self._check_base_destructor(self.TextIOBase)
755
756 def test_close_flushes(self):
757 with self.open(os_helper.TESTFN, "wb") as f:
758 f.write(b"xxx")
759 with self.open(os_helper.TESTFN, "rb") as f:
760 self.assertEqual(f.read(), b"xxx")
761
762 def test_array_writes(self):
763 a = array.array('i', range(10))
764 n = len(a.tobytes())
765 def check(f):
766 with f:
767 self.assertEqual(f.write(a), n)
768 f.writelines((a,))
769 check(self.BytesIO())
770 check(self.FileIO(os_helper.TESTFN, "w"))
771 check(self.BufferedWriter(self.MockRawIO()))
772 check(self.BufferedRandom(self.MockRawIO()))
773 check(self.BufferedRWPair(self.MockRawIO(), self.MockRawIO()))
774
775 def test_closefd(self):
776 self.assertRaises(ValueError, self.open, os_helper.TESTFN, 'w',
777 encoding="utf-8", closefd=False)
778
779 def test_read_closed(self):
780 with self.open(os_helper.TESTFN, "w", encoding="utf-8") as f:
781 f.write("egg\n")
782 with self.open(os_helper.TESTFN, "r", encoding="utf-8") as f:
783 file = self.open(f.fileno(), "r", encoding="utf-8", closefd=False)
784 self.assertEqual(file.read(), "egg\n")
785 file.seek(0)
786 file.close()
787 self.assertRaises(ValueError, file.read)
788 with self.open(os_helper.TESTFN, "rb") as f:
789 file = self.open(f.fileno(), "rb", closefd=False)
790 self.assertEqual(file.read()[:3], b"egg")
791 file.close()
792 self.assertRaises(ValueError, file.readinto, bytearray(1))
793
794 def test_no_closefd_with_filename(self):
795 # can't use closefd in combination with a file name
796 self.assertRaises(ValueError, self.open, os_helper.TESTFN, "r",
797 encoding="utf-8", closefd=False)
798
799 def test_closefd_attr(self):
800 with self.open(os_helper.TESTFN, "wb") as f:
801 f.write(b"egg\n")
802 with self.open(os_helper.TESTFN, "r", encoding="utf-8") as f:
803 self.assertEqual(f.buffer.raw.closefd, True)
804 file = self.open(f.fileno(), "r", encoding="utf-8", closefd=False)
805 self.assertEqual(file.buffer.raw.closefd, False)
806
807 def test_garbage_collection(self):
808 # FileIO objects are collected, and collecting them flushes
809 # all data to disk.
810 with warnings_helper.check_warnings(('', ResourceWarning)):
811 f = self.FileIO(os_helper.TESTFN, "wb")
812 f.write(b"abcxxx")
813 f.f = f
814 wr = weakref.ref(f)
815 del f
816 support.gc_collect()
817 self.assertIsNone(wr(), wr)
818 with self.open(os_helper.TESTFN, "rb") as f:
819 self.assertEqual(f.read(), b"abcxxx")
820
821 def test_unbounded_file(self):
822 # Issue #1174606: reading from an unbounded stream such as /dev/zero.
823 zero = "/dev/zero"
824 if not os.path.exists(zero):
825 self.skipTest("{0} does not exist".format(zero))
826 if sys.maxsize > 0x7FFFFFFF:
827 self.skipTest("test can only run in a 32-bit address space")
828 if support.real_max_memuse < support._2G:
829 self.skipTest("test requires at least 2 GiB of memory")
830 with self.open(zero, "rb", buffering=0) as f:
831 self.assertRaises(OverflowError, f.read)
832 with self.open(zero, "rb") as f:
833 self.assertRaises(OverflowError, f.read)
834 with self.open(zero, "r") as f:
835 self.assertRaises(OverflowError, f.read)
836
837 def check_flush_error_on_close(self, *args, **kwargs):
838 # Test that the file is closed despite failed flush
839 # and that flush() is called before file closed.
840 f = self.open(*args, **kwargs)
841 closed = []
842 def bad_flush():
843 closed[:] = [f.closed]
844 raise OSError()
845 f.flush = bad_flush
846 self.assertRaises(OSError, f.close) # exception not swallowed
847 self.assertTrue(f.closed)
848 self.assertTrue(closed) # flush() called
849 self.assertFalse(closed[0]) # flush() called before file closed
850 f.flush = lambda: None # break reference loop
851
852 def test_flush_error_on_close(self):
853 # raw file
854 # Issue #5700: io.FileIO calls flush() after file closed
855 self.check_flush_error_on_close(os_helper.TESTFN, 'wb', buffering=0)
856 fd = os.open(os_helper.TESTFN, os.O_WRONLY|os.O_CREAT)
857 self.check_flush_error_on_close(fd, 'wb', buffering=0)
858 fd = os.open(os_helper.TESTFN, os.O_WRONLY|os.O_CREAT)
859 self.check_flush_error_on_close(fd, 'wb', buffering=0, closefd=False)
860 os.close(fd)
861 # buffered io
862 self.check_flush_error_on_close(os_helper.TESTFN, 'wb')
863 fd = os.open(os_helper.TESTFN, os.O_WRONLY|os.O_CREAT)
864 self.check_flush_error_on_close(fd, 'wb')
865 fd = os.open(os_helper.TESTFN, os.O_WRONLY|os.O_CREAT)
866 self.check_flush_error_on_close(fd, 'wb', closefd=False)
867 os.close(fd)
868 # text io
869 self.check_flush_error_on_close(os_helper.TESTFN, 'w', encoding="utf-8")
870 fd = os.open(os_helper.TESTFN, os.O_WRONLY|os.O_CREAT)
871 self.check_flush_error_on_close(fd, 'w', encoding="utf-8")
872 fd = os.open(os_helper.TESTFN, os.O_WRONLY|os.O_CREAT)
873 self.check_flush_error_on_close(fd, 'w', encoding="utf-8", closefd=False)
874 os.close(fd)
875
876 def test_multi_close(self):
877 f = self.open(os_helper.TESTFN, "wb", buffering=0)
878 f.close()
879 f.close()
880 f.close()
881 self.assertRaises(ValueError, f.flush)
882
883 def test_RawIOBase_read(self):
884 # Exercise the default limited RawIOBase.read(n) implementation (which
885 # calls readinto() internally).
886 rawio = self.MockRawIOWithoutRead((b"abc", b"d", None, b"efg", None))
887 self.assertEqual(rawio.read(2), b"ab")
888 self.assertEqual(rawio.read(2), b"c")
889 self.assertEqual(rawio.read(2), b"d")
890 self.assertEqual(rawio.read(2), None)
891 self.assertEqual(rawio.read(2), b"ef")
892 self.assertEqual(rawio.read(2), b"g")
893 self.assertEqual(rawio.read(2), None)
894 self.assertEqual(rawio.read(2), b"")
895
896 def test_RawIOBase_read_bounds_checking(self):
897 # Make sure a `.readinto` call which returns a value outside
898 # (0, len(buffer)) raises.
899 class Misbehaved(self.RawIOBase):
900 def __init__(self, readinto_return) -> None:
901 self._readinto_return = readinto_return
902 def readinto(self, b):
903 return self._readinto_return
904
905 with self.assertRaises(ValueError) as cm:
906 Misbehaved(2).read(1)
907 self.assertEqual(str(cm.exception), "readinto returned 2 outside buffer size 1")
908 for bad_size in (2147483647, sys.maxsize, -1, -1000):
909 with self.assertRaises(ValueError):
910 Misbehaved(bad_size).read()
911
912 def test_types_have_dict(self):
913 test = (
914 self.IOBase(),
915 self.RawIOBase(),
916 self.TextIOBase(),
917 self.StringIO(),
918 self.BytesIO()
919 )
920 for obj in test:
921 self.assertHasAttr(obj, "__dict__")
922
923 def test_opener(self):
924 with self.open(os_helper.TESTFN, "w", encoding="utf-8") as f:
925 f.write("egg\n")
926 fd = os.open(os_helper.TESTFN, os.O_RDONLY)
927 def opener(path, flags):
928 return fd
929 with self.open("non-existent", "r", encoding="utf-8", opener=opener) as f:
930 self.assertEqual(f.read(), "egg\n")
931
932 def test_bad_opener_negative_1(self):
933 # Issue #27066.
934 def badopener(fname, flags):
935 return -1
936 with self.assertRaises(ValueError) as cm:
937 self.open('non-existent', 'r', opener=badopener)
938 self.assertEqual(str(cm.exception), 'opener returned -1')
939
940 def test_bad_opener_other_negative(self):
941 # Issue #27066.
942 def badopener(fname, flags):
943 return -2
944 with self.assertRaises(ValueError) as cm:
945 self.open('non-existent', 'r', opener=badopener)
946 self.assertEqual(str(cm.exception), 'opener returned -2')
947
948 def test_opener_invalid_fd(self):
949 # Check that OSError is raised with error code EBADF if the
950 # opener returns an invalid file descriptor (see gh-82212).
951 fd = os_helper.make_bad_fd()
952 with self.assertRaises(OSError) as cm:
953 self.open('foo', opener=lambda name, flags: fd)
954 self.assertEqual(cm.exception.errno, errno.EBADF)
955
956 def test_fileio_closefd(self):
957 # Issue #4841
958 with self.open(__file__, 'rb') as f1, \
959 self.open(__file__, 'rb') as f2:
960 fileio = self.FileIO(f1.fileno(), closefd=False)
961 # .__init__() must not close f1
962 fileio.__init__(f2.fileno(), closefd=False)
963 f1.readline()
964 # .close() must not close f2
965 fileio.close()
966 f2.readline()
967
968 def test_nonbuffered_textio(self):
969 with warnings_helper.check_no_resource_warning(self):
970 with self.assertRaises(ValueError):
971 self.open(os_helper.TESTFN, 'w', encoding="utf-8", buffering=0)
972
973 def test_invalid_newline(self):
974 with warnings_helper.check_no_resource_warning(self):
975 with self.assertRaises(ValueError):
976 self.open(os_helper.TESTFN, 'w', encoding="utf-8", newline='invalid')
977
978 def test_buffered_readinto_mixin(self):
979 # Test the implementation provided by BufferedIOBase
980 class Stream(self.BufferedIOBase):
981 def read(self, size):
982 return b"12345"
983 read1 = read
984 stream = Stream()
985 for method in ("readinto", "readinto1"):
986 with self.subTest(method):
987 buffer = byteslike(5)
988 self.assertEqual(getattr(stream, method)(buffer), 5)
989 self.assertEqual(bytes(buffer), b"12345")
990
991 def test_fspath_support(self):
992 def check_path_succeeds(path):
993 with self.open(path, "w", encoding="utf-8") as f:
994 f.write("egg\n")
995
996 with self.open(path, "r", encoding="utf-8") as f:
997 self.assertEqual(f.read(), "egg\n")
998
999 check_path_succeeds(FakePath(os_helper.TESTFN))
1000 check_path_succeeds(FakePath(os.fsencode(os_helper.TESTFN)))
1001
1002 with self.open(os_helper.TESTFN, "w", encoding="utf-8") as f:
1003 bad_path = FakePath(f.fileno())
1004 with self.assertRaises(TypeError):
1005 self.open(bad_path, 'w', encoding="utf-8")
1006
1007 bad_path = FakePath(None)
1008 with self.assertRaises(TypeError):
1009 self.open(bad_path, 'w', encoding="utf-8")
1010
1011 bad_path = FakePath(FloatingPointError)
1012 with self.assertRaises(FloatingPointError):
1013 self.open(bad_path, 'w', encoding="utf-8")
1014
1015 # ensure that refcounting is correct with some error conditions
1016 with self.assertRaisesRegex(ValueError, 'read/write/append mode'):
1017 self.open(FakePath(os_helper.TESTFN), 'rwxa', encoding="utf-8")
1018
1019 def test_RawIOBase_readall(self):
1020 # Exercise the default unlimited RawIOBase.read() and readall()
1021 # implementations.
1022 rawio = self.MockRawIOWithoutRead((b"abc", b"d", b"efg"))
1023 self.assertEqual(rawio.read(), b"abcdefg")
1024 rawio = self.MockRawIOWithoutRead((b"abc", b"d", b"efg"))
1025 self.assertEqual(rawio.readall(), b"abcdefg")
1026
1027 def test_BufferedIOBase_readinto(self):
1028 # Exercise the default BufferedIOBase.readinto() and readinto1()
1029 # implementations (which call read() or read1() internally).
1030 class Reader(self.BufferedIOBase):
1031 def __init__(self, avail):
1032 self.avail = avail
1033 def read(self, size):
1034 result = self.avail[:size]
1035 self.avail = self.avail[size:]
1036 return result
1037 def read1(self, size):
1038 """Returns no more than 5 bytes at once"""
1039 return self.read(min(size, 5))
1040 tests = (
1041 # (test method, total data available, read buffer size, expected
1042 # read size)
1043 ("readinto", 10, 5, 5),
1044 ("readinto", 10, 6, 6), # More than read1() can return
1045 ("readinto", 5, 6, 5), # Buffer larger than total available
1046 ("readinto", 6, 7, 6),
1047 ("readinto", 10, 0, 0), # Empty buffer
1048 ("readinto1", 10, 5, 5), # Result limited to single read1() call
1049 ("readinto1", 10, 6, 5), # Buffer larger than read1() can return
1050 ("readinto1", 5, 6, 5), # Buffer larger than total available
1051 ("readinto1", 6, 7, 5),
1052 ("readinto1", 10, 0, 0), # Empty buffer
1053 )
1054 UNUSED_BYTE = 0x81
1055 for test in tests:
1056 with self.subTest(test):
1057 method, avail, request, result = test
1058 reader = Reader(bytes(range(avail)))
1059 buffer = bytearray((UNUSED_BYTE,) * request)
1060 method = getattr(reader, method)
1061 self.assertEqual(method(buffer), result)
1062 self.assertEqual(len(buffer), request)
1063 self.assertSequenceEqual(buffer[:result], range(result))
1064 unused = (UNUSED_BYTE,) * (request - result)
1065 self.assertSequenceEqual(buffer[result:], unused)
1066 self.assertEqual(len(reader.avail), avail - result)
1067
1068 def test_close_assert(self):
1069 class R(self.IOBase):
1070 def __setattr__(self, name, value):
1071 pass
1072 def flush(self):
1073 raise OSError()
1074 f = R()
1075 # This would cause an assertion failure.
1076 self.assertRaises(OSError, f.close)
1077
1078 # Silence destructor error
1079 R.flush = lambda self: None
1080
1081 @threading_helper.requires_working_threading()
1082 def test_write_readline_races(self):
1083 # gh-134908: Concurrent iteration over a file caused races
1084 thread_count = 2
1085 write_count = 100
1086 read_count = 100
1087
1088 def writer(file, barrier):
1089 barrier.wait()
1090 for _ in range(write_count):
1091 file.write("x")
1092
1093 def reader(file, barrier):
1094 barrier.wait()
1095 for _ in range(read_count):
1096 for line in file:
1097 self.assertEqual(line, "")
1098
1099 with self.open(os_helper.TESTFN, "w+") as f:
1100 barrier = threading.Barrier(thread_count + 1)
1101 reader = threading.Thread(target=reader, args=(f, barrier))
1102 writers = [threading.Thread(target=writer, args=(f, barrier))
1103 for _ in range(thread_count)]
1104 with threading_helper.catch_threading_exception() as cm:
1105 with threading_helper.start_threads(writers + [reader]):
1106 pass
1107 self.assertIsNone(cm.exc_type)
1108
1109 self.assertEqual(os.stat(os_helper.TESTFN).st_size,
1110 write_count * thread_count)
1111
1112
1113 class CIOTest(IOTest):
1114
1115 def test_IOBase_finalize(self):
1116 # Issue #12149: segmentation fault on _PyIOBase_finalize when both a
1117 # class which inherits IOBase and an object of this class are caught
1118 # in a reference cycle and close() is already in the method cache.
1119 class MyIO(self.IOBase):
1120 def close(self):
1121 pass
1122
1123 # create an instance to populate the method cache
1124 MyIO()
1125 obj = MyIO()
1126 obj.obj = obj
1127 wr = weakref.ref(obj)
1128 del MyIO
1129 del obj
1130 support.gc_collect()
1131 self.assertIsNone(wr(), wr)
1132
1133 @support.cpython_only
1134 class TestIOCTypes(unittest.TestCase):
1135 def setUp(self):
1136 _io = import_helper.import_module("_io")
1137 self.types = [
1138 _io.BufferedRWPair,
1139 _io.BufferedRandom,
1140 _io.BufferedReader,
1141 _io.BufferedWriter,
1142 _io.BytesIO,
1143 _io.FileIO,
1144 _io.IncrementalNewlineDecoder,
1145 _io.StringIO,
1146 _io.TextIOWrapper,
1147 _io._BufferedIOBase,
1148 _io._BytesIOBuffer,
1149 _io._IOBase,
1150 _io._RawIOBase,
1151 _io._TextIOBase,
1152 ]
1153 if sys.platform == "win32":
1154 self.types.append(_io._WindowsConsoleIO)
1155 self._io = _io
1156
1157 def test_immutable_types(self):
1158 for tp in self.types:
1159 with self.subTest(tp=tp):
1160 with self.assertRaisesRegex(TypeError, "immutable"):
1161 tp.foo = "bar"
1162
1163 def test_class_hierarchy(self):
1164 def check_subs(types, base):
1165 for tp in types:
1166 with self.subTest(tp=tp, base=base):
1167 self.assertIsSubclass(tp, base)
1168
1169 def recursive_check(d):
1170 for k, v in d.items():
1171 if isinstance(v, dict):
1172 recursive_check(v)
1173 elif isinstance(v, set):
1174 check_subs(v, k)
1175 else:
1176 self.fail("corrupt test dataset")
1177
1178 _io = self._io
1179 hierarchy = {
1180 _io._IOBase: {
1181 _io._BufferedIOBase: {
1182 _io.BufferedRWPair,
1183 _io.BufferedRandom,
1184 _io.BufferedReader,
1185 _io.BufferedWriter,
1186 _io.BytesIO,
1187 },
1188 _io._RawIOBase: {
1189 _io.FileIO,
1190 },
1191 _io._TextIOBase: {
1192 _io.StringIO,
1193 _io.TextIOWrapper,
1194 },
1195 },
1196 }
1197 if sys.platform == "win32":
1198 hierarchy[_io._IOBase][_io._RawIOBase].add(_io._WindowsConsoleIO)
1199
1200 recursive_check(hierarchy)
1201
1202 def test_subclassing(self):
1203 _io = self._io
1204 dataset = {k: True for k in self.types}
1205 dataset[_io._BytesIOBuffer] = False
1206
1207 for tp, is_basetype in dataset.items():
1208 with self.subTest(tp=tp, is_basetype=is_basetype):
1209 name = f"{tp.__name__}_subclass"
1210 bases = (tp,)
1211 if is_basetype:
1212 _ = type(name, bases, {})
1213 else:
1214 msg = "not an acceptable base type"
1215 with self.assertRaisesRegex(TypeError, msg):
1216 _ = type(name, bases, {})
1217
1218 def test_disallow_instantiation(self):
1219 _io = self._io
1220 support.check_disallow_instantiation(self, _io._BytesIOBuffer)
1221
1222 def test_stringio_setstate(self):
1223 # gh-127182: Calling __setstate__() with invalid arguments must not crash
1224 obj = self._io.StringIO()
1225 with self.assertRaisesRegex(
1226 TypeError,
1227 'initial_value must be str or None, not int',
1228 ):
1229 obj.__setstate__((1, '', 0, {}))
1230
1231 obj.__setstate__((None, '', 0, {})) # should not crash
1232 self.assertEqual(obj.getvalue(), '')
1233
1234 obj.__setstate__(('', '', 0, {}))
1235 self.assertEqual(obj.getvalue(), '')
1236
1237 class PyIOTest(IOTest):
1238 pass
1239
1240
1241 @support.cpython_only
1242 class APIMismatchTest(unittest.TestCase):
1243
1244 def test_RawIOBase_io_in_pyio_match(self):
1245 """Test that pyio RawIOBase class has all c RawIOBase methods"""
1246 mismatch = support.detect_api_mismatch(pyio.RawIOBase, io.RawIOBase,
1247 ignore=('__weakref__', '__static_attributes__'))
1248 self.assertEqual(mismatch, set(), msg='Python RawIOBase does not have all C RawIOBase methods')
1249
1250 def test_RawIOBase_pyio_in_io_match(self):
1251 """Test that c RawIOBase class has all pyio RawIOBase methods"""
1252 mismatch = support.detect_api_mismatch(io.RawIOBase, pyio.RawIOBase)
1253 self.assertEqual(mismatch, set(), msg='C RawIOBase does not have all Python RawIOBase methods')
1254
1255
1256 class CommonBufferedTests:
1257 # Tests common to BufferedReader, BufferedWriter and BufferedRandom
1258
1259 def test_detach(self):
1260 raw = self.MockRawIO()
1261 buf = self.tp(raw)
1262 self.assertIs(buf.detach(), raw)
1263 self.assertRaises(ValueError, buf.detach)
1264
1265 repr(buf) # Should still work
1266
1267 def test_fileno(self):
1268 rawio = self.MockRawIO()
1269 bufio = self.tp(rawio)
1270
1271 self.assertEqual(42, bufio.fileno())
1272
1273 def test_invalid_args(self):
1274 rawio = self.MockRawIO()
1275 bufio = self.tp(rawio)
1276 # Invalid whence
1277 self.assertRaises(ValueError, bufio.seek, 0, -1)
1278 self.assertRaises(ValueError, bufio.seek, 0, 9)
1279
1280 def test_override_destructor(self):
1281 tp = self.tp
1282 record = []
1283 class MyBufferedIO(tp):
1284 def __del__(self):
1285 record.append(1)
1286 try:
1287 f = super().__del__
1288 except AttributeError:
1289 pass
1290 else:
1291 f()
1292 def close(self):
1293 record.append(2)
1294 super().close()
1295 def flush(self):
1296 record.append(3)
1297 super().flush()
1298 rawio = self.MockRawIO()
1299 bufio = MyBufferedIO(rawio)
1300 del bufio
1301 support.gc_collect()
1302 self.assertEqual(record, [1, 2, 3])
1303
1304 def test_context_manager(self):
1305 # Test usability as a context manager
1306 rawio = self.MockRawIO()
1307 bufio = self.tp(rawio)
1308 def _with():
1309 with bufio:
1310 pass
1311 _with()
1312 # bufio should now be closed, and using it a second time should raise
1313 # a ValueError.
1314 self.assertRaises(ValueError, _with)
1315
1316 def test_error_through_destructor(self):
1317 # Test that the exception state is not modified by a destructor,
1318 # even if close() fails.
1319 rawio = self.CloseFailureIO()
1320 with support.catch_unraisable_exception() as cm:
1321 with self.assertRaises(AttributeError):
1322 self.tp(rawio).xyzzy
1323
1324 self.assertEqual(cm.unraisable.exc_type, OSError)
1325
1326 def test_repr(self):
1327 raw = self.MockRawIO()
1328 b = self.tp(raw)
1329 clsname = r"(%s\.)?%s" % (self.tp.__module__, self.tp.__qualname__)
1330 self.assertRegex(repr(b), "<%s>" % clsname)
1331 raw.name = "dummy"
1332 self.assertRegex(repr(b), "<%s name='dummy'>" % clsname)
1333 raw.name = b"dummy"
1334 self.assertRegex(repr(b), "<%s name=b'dummy'>" % clsname)
1335
1336 def test_recursive_repr(self):
1337 # Issue #25455
1338 raw = self.MockRawIO()
1339 b = self.tp(raw)
1340 with support.swap_attr(raw, 'name', b), support.infinite_recursion(25):
1341 with self.assertRaises(RuntimeError):
1342 repr(b) # Should not crash
1343
1344 def test_flush_error_on_close(self):
1345 # Test that buffered file is closed despite failed flush
1346 # and that flush() is called before file closed.
1347 raw = self.MockRawIO()
1348 closed = []
1349 def bad_flush():
1350 closed[:] = [b.closed, raw.closed]
1351 raise OSError()
1352 raw.flush = bad_flush
1353 b = self.tp(raw)
1354 self.assertRaises(OSError, b.close) # exception not swallowed
1355 self.assertTrue(b.closed)
1356 self.assertTrue(raw.closed)
1357 self.assertTrue(closed) # flush() called
1358 self.assertFalse(closed[0]) # flush() called before file closed
1359 self.assertFalse(closed[1])
1360 raw.flush = lambda: None # break reference loop
1361
1362 def test_close_error_on_close(self):
1363 raw = self.MockRawIO()
1364 def bad_flush():
1365 raise OSError('flush')
1366 def bad_close():
1367 raise OSError('close')
1368 raw.close = bad_close
1369 b = self.tp(raw)
1370 b.flush = bad_flush
1371 with self.assertRaises(OSError) as err: # exception not swallowed
1372 b.close()
1373 self.assertEqual(err.exception.args, ('close',))
1374 self.assertIsInstance(err.exception.__context__, OSError)
1375 self.assertEqual(err.exception.__context__.args, ('flush',))
1376 self.assertFalse(b.closed)
1377
1378 # Silence destructor error
1379 raw.close = lambda: None
1380 b.flush = lambda: None
1381
1382 def test_nonnormalized_close_error_on_close(self):
1383 # Issue #21677
1384 raw = self.MockRawIO()
1385 def bad_flush():
1386 raise non_existing_flush
1387 def bad_close():
1388 raise non_existing_close
1389 raw.close = bad_close
1390 b = self.tp(raw)
1391 b.flush = bad_flush
1392 with self.assertRaises(NameError) as err: # exception not swallowed
1393 b.close()
1394 self.assertIn('non_existing_close', str(err.exception))
1395 self.assertIsInstance(err.exception.__context__, NameError)
1396 self.assertIn('non_existing_flush', str(err.exception.__context__))
1397 self.assertFalse(b.closed)
1398
1399 # Silence destructor error
1400 b.flush = lambda: None
1401 raw.close = lambda: None
1402
1403 def test_multi_close(self):
1404 raw = self.MockRawIO()
1405 b = self.tp(raw)
1406 b.close()
1407 b.close()
1408 b.close()
1409 self.assertRaises(ValueError, b.flush)
1410
1411 def test_unseekable(self):
1412 bufio = self.tp(self.MockUnseekableIO(b"A" * 10))
1413 self.assertRaises(self.UnsupportedOperation, bufio.tell)
1414 self.assertRaises(self.UnsupportedOperation, bufio.seek, 0)
1415
1416 def test_readonly_attributes(self):
1417 raw = self.MockRawIO()
1418 buf = self.tp(raw)
1419 x = self.MockRawIO()
1420 with self.assertRaises(AttributeError):
1421 buf.raw = x
1422
1423 def test_pickling_subclass(self):
1424 global MyBufferedIO
1425 class MyBufferedIO(self.tp):
1426 def __init__(self, raw, tag):
1427 super().__init__(raw)
1428 self.tag = tag
1429 def __getstate__(self):
1430 return self.tag, self.raw.getvalue()
1431 def __setstate__(slf, state):
1432 tag, value = state
1433 slf.__init__(self.BytesIO(value), tag)
1434
1435 raw = self.BytesIO(b'data')
1436 buf = MyBufferedIO(raw, tag='ham')
1437 for proto in range(pickle.HIGHEST_PROTOCOL + 1):
1438 with self.subTest(protocol=proto):
1439 pickled = pickle.dumps(buf, proto)
1440 newbuf = pickle.loads(pickled)
1441 self.assertEqual(newbuf.raw.getvalue(), b'data')
1442 self.assertEqual(newbuf.tag, 'ham')
1443 del MyBufferedIO
1444
1445
1446 class SizeofTest:
1447
1448 @support.cpython_only
1449 def test_sizeof(self):
1450 bufsize1 = 4096
1451 bufsize2 = 8192
1452 rawio = self.MockRawIO()
1453 bufio = self.tp(rawio, buffer_size=bufsize1)
1454 size = sys.getsizeof(bufio) - bufsize1
1455 rawio = self.MockRawIO()
1456 bufio = self.tp(rawio, buffer_size=bufsize2)
1457 self.assertEqual(sys.getsizeof(bufio), size + bufsize2)
1458
1459 @support.cpython_only
1460 def test_buffer_freeing(self) :
1461 bufsize = 4096
1462 rawio = self.MockRawIO()
1463 bufio = self.tp(rawio, buffer_size=bufsize)
1464 size = sys.getsizeof(bufio) - bufsize
1465 bufio.close()
1466 self.assertEqual(sys.getsizeof(bufio), size)
1467
1468 class BufferedReaderTest(unittest.TestCase, CommonBufferedTests):
1469 read_mode = "rb"
1470
1471 def test_constructor(self):
1472 rawio = self.MockRawIO([b"abc"])
1473 bufio = self.tp(rawio)
1474 bufio.__init__(rawio)
1475 bufio.__init__(rawio, buffer_size=1024)
1476 bufio.__init__(rawio, buffer_size=16)
1477 self.assertEqual(b"abc", bufio.read())
1478 self.assertRaises(ValueError, bufio.__init__, rawio, buffer_size=0)
1479 self.assertRaises(ValueError, bufio.__init__, rawio, buffer_size=-16)
1480 self.assertRaises(ValueError, bufio.__init__, rawio, buffer_size=-1)
1481 rawio = self.MockRawIO([b"abc"])
1482 bufio.__init__(rawio)
1483 self.assertEqual(b"abc", bufio.read())
1484
1485 def test_uninitialized(self):
1486 bufio = self.tp.__new__(self.tp)
1487 del bufio
1488 bufio = self.tp.__new__(self.tp)
1489 self.assertRaisesRegex((ValueError, AttributeError),
1490 'uninitialized|has no attribute',
1491 bufio.read, 0)
1492 bufio.__init__(self.MockRawIO())
1493 self.assertEqual(bufio.read(0), b'')
1494
1495 def test_read(self):
1496 for arg in (None, 7):
1497 rawio = self.MockRawIO((b"abc", b"d", b"efg"))
1498 bufio = self.tp(rawio)
1499 self.assertEqual(b"abcdefg", bufio.read(arg))
1500 # Invalid args
1501 self.assertRaises(ValueError, bufio.read, -2)
1502
1503 def test_read1(self):
1504 rawio = self.MockRawIO((b"abc", b"d", b"efg"))
1505 bufio = self.tp(rawio)
1506 self.assertEqual(b"a", bufio.read(1))
1507 self.assertEqual(b"b", bufio.read1(1))
1508 self.assertEqual(rawio._reads, 1)
1509 self.assertEqual(b"", bufio.read1(0))
1510 self.assertEqual(b"c", bufio.read1(100))
1511 self.assertEqual(rawio._reads, 1)
1512 self.assertEqual(b"d", bufio.read1(100))
1513 self.assertEqual(rawio._reads, 2)
1514 self.assertEqual(b"efg", bufio.read1(100))
1515 self.assertEqual(rawio._reads, 3)
1516 self.assertEqual(b"", bufio.read1(100))
1517 self.assertEqual(rawio._reads, 4)
1518
1519 def test_read1_arbitrary(self):
1520 rawio = self.MockRawIO((b"abc", b"d", b"efg"))
1521 bufio = self.tp(rawio)
1522 self.assertEqual(b"a", bufio.read(1))
1523 self.assertEqual(b"bc", bufio.read1())
1524 self.assertEqual(b"d", bufio.read1())
1525 self.assertEqual(b"efg", bufio.read1(-1))
1526 self.assertEqual(rawio._reads, 3)
1527 self.assertEqual(b"", bufio.read1())
1528 self.assertEqual(rawio._reads, 4)
1529
1530 def test_readinto(self):
1531 rawio = self.MockRawIO((b"abc", b"d", b"efg"))
1532 bufio = self.tp(rawio)
1533 b = bytearray(2)
1534 self.assertEqual(bufio.readinto(b), 2)
1535 self.assertEqual(b, b"ab")
1536 self.assertEqual(bufio.readinto(b), 2)
1537 self.assertEqual(b, b"cd")
1538 self.assertEqual(bufio.readinto(b), 2)
1539 self.assertEqual(b, b"ef")
1540 self.assertEqual(bufio.readinto(b), 1)
1541 self.assertEqual(b, b"gf")
1542 self.assertEqual(bufio.readinto(b), 0)
1543 self.assertEqual(b, b"gf")
1544 rawio = self.MockRawIO((b"abc", None))
1545 bufio = self.tp(rawio)
1546 self.assertEqual(bufio.readinto(b), 2)
1547 self.assertEqual(b, b"ab")
1548 self.assertEqual(bufio.readinto(b), 1)
1549 self.assertEqual(b, b"cb")
1550
1551 def test_readinto1(self):
1552 buffer_size = 10
1553 rawio = self.MockRawIO((b"abc", b"de", b"fgh", b"jkl"))
1554 bufio = self.tp(rawio, buffer_size=buffer_size)
1555 b = bytearray(2)
1556 self.assertEqual(bufio.peek(3), b'abc')
1557 self.assertEqual(rawio._reads, 1)
1558 self.assertEqual(bufio.readinto1(b), 2)
1559 self.assertEqual(b, b"ab")
1560 self.assertEqual(rawio._reads, 1)
1561 self.assertEqual(bufio.readinto1(b), 1)
1562 self.assertEqual(b[:1], b"c")
1563 self.assertEqual(rawio._reads, 1)
1564 self.assertEqual(bufio.readinto1(b), 2)
1565 self.assertEqual(b, b"de")
1566 self.assertEqual(rawio._reads, 2)
1567 b = bytearray(2*buffer_size)
1568 self.assertEqual(bufio.peek(3), b'fgh')
1569 self.assertEqual(rawio._reads, 3)
1570 self.assertEqual(bufio.readinto1(b), 6)
1571 self.assertEqual(b[:6], b"fghjkl")
1572 self.assertEqual(rawio._reads, 4)
1573
1574 def test_readinto_array(self):
1575 buffer_size = 60
1576 data = b"a" * 26
1577 rawio = self.MockRawIO((data,))
1578 bufio = self.tp(rawio, buffer_size=buffer_size)
1579
1580 # Create an array with element size > 1 byte
1581 b = array.array('i', b'x' * 32)
1582 assert len(b) != 16
1583
1584 # Read into it. We should get as many *bytes* as we can fit into b
1585 # (which is more than the number of elements)
1586 n = bufio.readinto(b)
1587 self.assertGreater(n, len(b))
1588
1589 # Check that old contents of b are preserved
1590 bm = memoryview(b).cast('B')
1591 self.assertLess(n, len(bm))
1592 self.assertEqual(bm[:n], data[:n])
1593 self.assertEqual(bm[n:], b'x' * (len(bm[n:])))
1594
1595 def test_readinto1_array(self):
1596 buffer_size = 60
1597 data = b"a" * 26
1598 rawio = self.MockRawIO((data,))
1599 bufio = self.tp(rawio, buffer_size=buffer_size)
1600
1601 # Create an array with element size > 1 byte
1602 b = array.array('i', b'x' * 32)
1603 assert len(b) != 16
1604
1605 # Read into it. We should get as many *bytes* as we can fit into b
1606 # (which is more than the number of elements)
1607 n = bufio.readinto1(b)
1608 self.assertGreater(n, len(b))
1609
1610 # Check that old contents of b are preserved
1611 bm = memoryview(b).cast('B')
1612 self.assertLess(n, len(bm))
1613 self.assertEqual(bm[:n], data[:n])
1614 self.assertEqual(bm[n:], b'x' * (len(bm[n:])))
1615
1616 def test_readlines(self):
1617 def bufio():
1618 rawio = self.MockRawIO((b"abc\n", b"d\n", b"ef"))
1619 return self.tp(rawio)
1620 self.assertEqual(bufio().readlines(), [b"abc\n", b"d\n", b"ef"])
1621 self.assertEqual(bufio().readlines(5), [b"abc\n", b"d\n"])
1622 self.assertEqual(bufio().readlines(None), [b"abc\n", b"d\n", b"ef"])
1623
1624 def test_buffering(self):
1625 data = b"abcdefghi"
1626 dlen = len(data)
1627
1628 tests = [
1629 [ 100, [ 3, 1, 4, 8 ], [ dlen, 0 ] ],
1630 [ 100, [ 3, 3, 3], [ dlen ] ],
1631 [ 4, [ 1, 2, 4, 2 ], [ 4, 4, 1 ] ],
1632 ]
1633
1634 for bufsize, buf_read_sizes, raw_read_sizes in tests:
1635 rawio = self.MockFileIO(data)
1636 bufio = self.tp(rawio, buffer_size=bufsize)
1637 pos = 0
1638 for nbytes in buf_read_sizes:
1639 self.assertEqual(bufio.read(nbytes), data[pos:pos+nbytes])
1640 pos += nbytes
1641 # this is mildly implementation-dependent
1642 self.assertEqual(rawio.read_history, raw_read_sizes)
1643
1644 def test_read_non_blocking(self):
1645 # Inject some None's in there to simulate EWOULDBLOCK
1646 rawio = self.MockRawIO((b"abc", b"d", None, b"efg", None, None, None))
1647 bufio = self.tp(rawio)
1648 self.assertEqual(b"abcd", bufio.read(6))
1649 self.assertEqual(b"e", bufio.read(1))
1650 self.assertEqual(b"fg", bufio.read())
1651 self.assertEqual(b"", bufio.peek(1))
1652 self.assertIsNone(bufio.read())
1653 self.assertEqual(b"", bufio.read())
1654
1655 rawio = self.MockRawIO((b"a", None, None))
1656 self.assertEqual(b"a", rawio.readall())
1657 self.assertIsNone(rawio.readall())
1658
1659 def test_read_past_eof(self):
1660 rawio = self.MockRawIO((b"abc", b"d", b"efg"))
1661 bufio = self.tp(rawio)
1662
1663 self.assertEqual(b"abcdefg", bufio.read(9000))
1664
1665 def test_read_all(self):
1666 rawio = self.MockRawIO((b"abc", b"d", b"efg"))
1667 bufio = self.tp(rawio)
1668
1669 self.assertEqual(b"abcdefg", bufio.read())
1670
1671 @threading_helper.requires_working_threading()
1672 @support.requires_resource('cpu')
1673 def test_threads(self):
1674 try:
1675 # Write out many bytes with exactly the same number of 0's,
1676 # 1's... 255's. This will help us check that concurrent reading
1677 # doesn't duplicate or forget contents.
1678 N = 1000
1679 l = list(range(256)) * N
1680 random.shuffle(l)
1681 s = bytes(bytearray(l))
1682 with self.open(os_helper.TESTFN, "wb") as f:
1683 f.write(s)
1684 with self.open(os_helper.TESTFN, self.read_mode, buffering=0) as raw:
1685 bufio = self.tp(raw, 8)
1686 errors = []
1687 results = []
1688 def f():
1689 try:
1690 # Intra-buffer read then buffer-flushing read
1691 for n in cycle([1, 19]):
1692 s = bufio.read(n)
1693 if not s:
1694 break
1695 # list.append() is atomic
1696 results.append(s)
1697 except Exception as e:
1698 errors.append(e)
1699 raise
1700 threads = [threading.Thread(target=f) for x in range(20)]
1701 with threading_helper.start_threads(threads):
1702 time.sleep(0.02) # yield
1703 self.assertFalse(errors,
1704 "the following exceptions were caught: %r" % errors)
1705 s = b''.join(results)
1706 for i in range(256):
1707 c = bytes(bytearray([i]))
1708 self.assertEqual(s.count(c), N)
1709 finally:
1710 os_helper.unlink(os_helper.TESTFN)
1711
1712 def test_unseekable(self):
1713 bufio = self.tp(self.MockUnseekableIO(b"A" * 10))
1714 self.assertRaises(self.UnsupportedOperation, bufio.tell)
1715 self.assertRaises(self.UnsupportedOperation, bufio.seek, 0)
1716 bufio.read(1)
1717 self.assertRaises(self.UnsupportedOperation, bufio.seek, 0)
1718 self.assertRaises(self.UnsupportedOperation, bufio.tell)
1719
1720 def test_misbehaved_io(self):
1721 rawio = self.MisbehavedRawIO((b"abc", b"d", b"efg"))
1722 bufio = self.tp(rawio)
1723 self.assertRaises(OSError, bufio.seek, 0)
1724 self.assertRaises(OSError, bufio.tell)
1725
1726 # Silence destructor error
1727 bufio.close = lambda: None
1728
1729 def test_no_extraneous_read(self):
1730 # Issue #9550; when the raw IO object has satisfied the read request,
1731 # we should not issue any additional reads, otherwise it may block
1732 # (e.g. socket).
1733 bufsize = 16
1734 for n in (2, bufsize - 1, bufsize, bufsize + 1, bufsize * 2):
1735 rawio = self.MockRawIO([b"x" * n])
1736 bufio = self.tp(rawio, bufsize)
1737 self.assertEqual(bufio.read(n), b"x" * n)
1738 # Simple case: one raw read is enough to satisfy the request.
1739 self.assertEqual(rawio._extraneous_reads, 0,
1740 "failed for {}: {} != 0".format(n, rawio._extraneous_reads))
1741 # A more complex case where two raw reads are needed to satisfy
1742 # the request.
1743 rawio = self.MockRawIO([b"x" * (n - 1), b"x"])
1744 bufio = self.tp(rawio, bufsize)
1745 self.assertEqual(bufio.read(n), b"x" * n)
1746 self.assertEqual(rawio._extraneous_reads, 0,
1747 "failed for {}: {} != 0".format(n, rawio._extraneous_reads))
1748
1749 def test_read_on_closed(self):
1750 # Issue #23796
1751 b = self.BufferedReader(self.BytesIO(b"12"))
1752 b.read(1)
1753 b.close()
1754 with self.subTest('peek'):
1755 self.assertRaises(ValueError, b.peek)
1756 with self.subTest('read1'):
1757 self.assertRaises(ValueError, b.read1, 1)
1758 with self.subTest('read'):
1759 self.assertRaises(ValueError, b.read)
1760 with self.subTest('readinto'):
1761 self.assertRaises(ValueError, b.readinto, bytearray())
1762 with self.subTest('readinto1'):
1763 self.assertRaises(ValueError, b.readinto1, bytearray())
1764 with self.subTest('flush'):
1765 self.assertRaises(ValueError, b.flush)
1766 with self.subTest('truncate'):
1767 self.assertRaises(ValueError, b.truncate)
1768 with self.subTest('seek'):
1769 self.assertRaises(ValueError, b.seek, 0)
1770
1771 def test_truncate_on_read_only(self):
1772 rawio = self.MockFileIO(b"abc")
1773 bufio = self.tp(rawio)
1774 self.assertFalse(bufio.writable())
1775 self.assertRaises(self.UnsupportedOperation, bufio.truncate)
1776 self.assertRaises(self.UnsupportedOperation, bufio.truncate, 0)
1777
1778 def test_tell_character_device_file(self):
1779 # GH-95782
1780 # For the (former) bug in BufferedIO to manifest, the wrapped IO obj
1781 # must be able to produce at least 2 bytes.
1782 raw = self.MockCharPseudoDevFileIO(b"12")
1783 buf = self.tp(raw)
1784 self.assertEqual(buf.tell(), 0)
1785 self.assertEqual(buf.read(1), b"1")
1786 self.assertEqual(buf.tell(), 0)
1787
1788 def test_seek_character_device_file(self):
1789 raw = self.MockCharPseudoDevFileIO(b"12")
1790 buf = self.tp(raw)
1791 self.assertEqual(buf.seek(0, io.SEEK_CUR), 0)
1792 self.assertEqual(buf.seek(1, io.SEEK_SET), 0)
1793 self.assertEqual(buf.seek(0, io.SEEK_CUR), 0)
1794 self.assertEqual(buf.read(1), b"1")
1795
1796 # In the C implementation, tell() sets the BufferedIO's abs_pos to 0,
1797 # which means that the next seek() could return a negative offset if it
1798 # does not sanity-check:
1799 self.assertEqual(buf.tell(), 0)
1800 self.assertEqual(buf.seek(0, io.SEEK_CUR), 0)
1801
1802
1803 class CBufferedReaderTest(BufferedReaderTest, SizeofTest):
1804 tp = io.BufferedReader
1805
1806 def test_initialization(self):
1807 rawio = self.MockRawIO([b"abc"])
1808 bufio = self.tp(rawio)
1809 self.assertRaises(ValueError, bufio.__init__, rawio, buffer_size=0)
1810 self.assertRaises(ValueError, bufio.read)
1811 self.assertRaises(ValueError, bufio.__init__, rawio, buffer_size=-16)
1812 self.assertRaises(ValueError, bufio.read)
1813 self.assertRaises(ValueError, bufio.__init__, rawio, buffer_size=-1)
1814 self.assertRaises(ValueError, bufio.read)
1815
1816 def test_misbehaved_io_read(self):
1817 rawio = self.MisbehavedRawIO((b"abc", b"d", b"efg"))
1818 bufio = self.tp(rawio)
1819 # _pyio.BufferedReader seems to implement reading different, so that
1820 # checking this is not so easy.
1821 self.assertRaises(OSError, bufio.read, 10)
1822
1823 def test_garbage_collection(self):
1824 # C BufferedReader objects are collected.
1825 # The Python version has __del__, so it ends into gc.garbage instead
1826 self.addCleanup(os_helper.unlink, os_helper.TESTFN)
1827 with warnings_helper.check_warnings(('', ResourceWarning)):
1828 rawio = self.FileIO(os_helper.TESTFN, "w+b")
1829 f = self.tp(rawio)
1830 f.f = f
1831 wr = weakref.ref(f)
1832 del f
1833 support.gc_collect()
1834 self.assertIsNone(wr(), wr)
1835
1836 def test_args_error(self):
1837 # Issue #17275
1838 with self.assertRaisesRegex(TypeError, "BufferedReader"):
1839 self.tp(self.BytesIO(), 1024, 1024, 1024)
1840
1841 def test_bad_readinto_value(self):
1842 rawio = self.tp(self.BytesIO(b"12"))
1843 rawio.readinto = lambda buf: -1
1844 bufio = self.tp(rawio)
1845 with self.assertRaises(OSError) as cm:
1846 bufio.readline()
1847 self.assertIsNone(cm.exception.__cause__)
1848
1849 def test_bad_readinto_type(self):
1850 rawio = self.tp(self.BytesIO(b"12"))
1851 rawio.readinto = lambda buf: b''
1852 bufio = self.tp(rawio)
1853 with self.assertRaises(OSError) as cm:
1854 bufio.readline()
1855 self.assertIsInstance(cm.exception.__cause__, TypeError)
1856
1857
1858 class PyBufferedReaderTest(BufferedReaderTest):
1859 tp = pyio.BufferedReader
1860
1861
1862 class BufferedWriterTest(unittest.TestCase, CommonBufferedTests):
1863 write_mode = "wb"
1864
1865 def test_constructor(self):
1866 rawio = self.MockRawIO()
1867 bufio = self.tp(rawio)
1868 bufio.__init__(rawio)
1869 bufio.__init__(rawio, buffer_size=1024)
1870 bufio.__init__(rawio, buffer_size=16)
1871 self.assertEqual(3, bufio.write(b"abc"))
1872 bufio.flush()
1873 self.assertRaises(ValueError, bufio.__init__, rawio, buffer_size=0)
1874 self.assertRaises(ValueError, bufio.__init__, rawio, buffer_size=-16)
1875 self.assertRaises(ValueError, bufio.__init__, rawio, buffer_size=-1)
1876 bufio.__init__(rawio)
1877 self.assertEqual(3, bufio.write(b"ghi"))
1878 bufio.flush()
1879 self.assertEqual(b"".join(rawio._write_stack), b"abcghi")
1880
1881 def test_uninitialized(self):
1882 bufio = self.tp.__new__(self.tp)
1883 del bufio
1884 bufio = self.tp.__new__(self.tp)
1885 self.assertRaisesRegex((ValueError, AttributeError),
1886 'uninitialized|has no attribute',
1887 bufio.write, b'')
1888 bufio.__init__(self.MockRawIO())
1889 self.assertEqual(bufio.write(b''), 0)
1890
1891 def test_detach_flush(self):
1892 raw = self.MockRawIO()
1893 buf = self.tp(raw)
1894 buf.write(b"howdy!")
1895 self.assertFalse(raw._write_stack)
1896 buf.detach()
1897 self.assertEqual(raw._write_stack, [b"howdy!"])
1898
1899 def test_write(self):
1900 # Write to the buffered IO but don't overflow the buffer.
1901 writer = self.MockRawIO()
1902 bufio = self.tp(writer, 8)
1903 bufio.write(b"abc")
1904 self.assertFalse(writer._write_stack)
1905 buffer = bytearray(b"def")
1906 bufio.write(buffer)
1907 buffer[:] = b"***" # Overwrite our copy of the data
1908 bufio.flush()
1909 self.assertEqual(b"".join(writer._write_stack), b"abcdef")
1910
1911 def test_write_overflow(self):
1912 writer = self.MockRawIO()
1913 bufio = self.tp(writer, 8)
1914 contents = b"abcdefghijklmnop"
1915 for n in range(0, len(contents), 3):
1916 bufio.write(contents[n:n+3])
1917 flushed = b"".join(writer._write_stack)
1918 # At least (total - 8) bytes were implicitly flushed, perhaps more
1919 # depending on the implementation.
1920 self.assertStartsWith(flushed, contents[:-8])
1921
1922 def check_writes(self, intermediate_func):
1923 # Lots of writes, test the flushed output is as expected.
1924 contents = bytes(range(256)) * 1000
1925 n = 0
1926 writer = self.MockRawIO()
1927 bufio = self.tp(writer, 13)
1928 # Generator of write sizes: repeat each N 15 times then proceed to N+1
1929 def gen_sizes():
1930 for size in count(1):
1931 for i in range(15):
1932 yield size
1933 sizes = gen_sizes()
1934 while n < len(contents):
1935 size = min(next(sizes), len(contents) - n)
1936 self.assertEqual(bufio.write(contents[n:n+size]), size)
1937 intermediate_func(bufio)
1938 n += size
1939 bufio.flush()
1940 self.assertEqual(contents, b"".join(writer._write_stack))
1941
1942 def test_writes(self):
1943 self.check_writes(lambda bufio: None)
1944
1945 def test_writes_and_flushes(self):
1946 self.check_writes(lambda bufio: bufio.flush())
1947
1948 def test_writes_and_seeks(self):
1949 def _seekabs(bufio):
1950 pos = bufio.tell()
1951 bufio.seek(pos + 1, 0)
1952 bufio.seek(pos - 1, 0)
1953 bufio.seek(pos, 0)
1954 self.check_writes(_seekabs)
1955 def _seekrel(bufio):
1956 pos = bufio.seek(0, 1)
1957 bufio.seek(+1, 1)
1958 bufio.seek(-1, 1)
1959 bufio.seek(pos, 0)
1960 self.check_writes(_seekrel)
1961
1962 def test_writes_and_truncates(self):
1963 self.check_writes(lambda bufio: bufio.truncate(bufio.tell()))
1964
1965 def test_write_non_blocking(self):
1966 raw = self.MockNonBlockWriterIO()
1967 bufio = self.tp(raw, 8)
1968
1969 self.assertEqual(bufio.write(b"abcd"), 4)
1970 self.assertEqual(bufio.write(b"efghi"), 5)
1971 # 1 byte will be written, the rest will be buffered
1972 raw.block_on(b"k")
1973 self.assertEqual(bufio.write(b"jklmn"), 5)
1974
1975 # 8 bytes will be written, 8 will be buffered and the rest will be lost
1976 raw.block_on(b"0")
1977 try:
1978 bufio.write(b"opqrwxyz0123456789")
1979 except self.BlockingIOError as e:
1980 written = e.characters_written
1981 else:
1982 self.fail("BlockingIOError should have been raised")
1983 self.assertEqual(written, 16)
1984 self.assertEqual(raw.pop_written(),
1985 b"abcdefghijklmnopqrwxyz")
1986
1987 self.assertEqual(bufio.write(b"ABCDEFGHI"), 9)
1988 s = raw.pop_written()
1989 # Previously buffered bytes were flushed
1990 self.assertStartsWith(s, b"01234567A")
1991
1992 def test_write_and_rewind(self):
1993 raw = self.BytesIO()
1994 bufio = self.tp(raw, 4)
1995 self.assertEqual(bufio.write(b"abcdef"), 6)
1996 self.assertEqual(bufio.tell(), 6)
1997 bufio.seek(0, 0)
1998 self.assertEqual(bufio.write(b"XY"), 2)
1999 bufio.seek(6, 0)
2000 self.assertEqual(raw.getvalue(), b"XYcdef")
2001 self.assertEqual(bufio.write(b"123456"), 6)
2002 bufio.flush()
2003 self.assertEqual(raw.getvalue(), b"XYcdef123456")
2004
2005 def test_flush(self):
2006 writer = self.MockRawIO()
2007 bufio = self.tp(writer, 8)
2008 bufio.write(b"abc")
2009 bufio.flush()
2010 self.assertEqual(b"abc", writer._write_stack[0])
2011
2012 def test_writelines(self):
2013 l = [b'ab', b'cd', b'ef']
2014 writer = self.MockRawIO()
2015 bufio = self.tp(writer, 8)
2016 bufio.writelines(l)
2017 bufio.flush()
2018 self.assertEqual(b''.join(writer._write_stack), b'abcdef')
2019
2020 def test_writelines_userlist(self):
2021 l = UserList([b'ab', b'cd', b'ef'])
2022 writer = self.MockRawIO()
2023 bufio = self.tp(writer, 8)
2024 bufio.writelines(l)
2025 bufio.flush()
2026 self.assertEqual(b''.join(writer._write_stack), b'abcdef')
2027
2028 def test_writelines_error(self):
2029 writer = self.MockRawIO()
2030 bufio = self.tp(writer, 8)
2031 self.assertRaises(TypeError, bufio.writelines, [1, 2, 3])
2032 self.assertRaises(TypeError, bufio.writelines, None)
2033 self.assertRaises(TypeError, bufio.writelines, 'abc')
2034
2035 def test_destructor(self):
2036 writer = self.MockRawIO()
2037 bufio = self.tp(writer, 8)
2038 bufio.write(b"abc")
2039 del bufio
2040 support.gc_collect()
2041 self.assertEqual(b"abc", writer._write_stack[0])
2042
2043 def test_truncate(self):
2044 # Truncate implicitly flushes the buffer.
2045 self.addCleanup(os_helper.unlink, os_helper.TESTFN)
2046 with self.open(os_helper.TESTFN, self.write_mode, buffering=0) as raw:
2047 bufio = self.tp(raw, 8)
2048 bufio.write(b"abcdef")
2049 self.assertEqual(bufio.truncate(3), 3)
2050 self.assertEqual(bufio.tell(), 6)
2051 with self.open(os_helper.TESTFN, "rb", buffering=0) as f:
2052 self.assertEqual(f.read(), b"abc")
2053
2054 def test_truncate_after_write(self):
2055 # Ensure that truncate preserves the file position after
2056 # writes longer than the buffer size.
2057 # Issue: https://bugs.python.org/issue32228
2058 self.addCleanup(os_helper.unlink, os_helper.TESTFN)
2059 with self.open(os_helper.TESTFN, "wb") as f:
2060 # Fill with some buffer
2061 f.write(b'\x00' * 10000)
2062 buffer_sizes = [8192, 4096, 200]
2063 for buffer_size in buffer_sizes:
2064 with self.open(os_helper.TESTFN, "r+b", buffering=buffer_size) as f:
2065 f.write(b'\x00' * (buffer_size + 1))
2066 # After write write_pos and write_end are set to 0
2067 f.read(1)
2068 # read operation makes sure that pos != raw_pos
2069 f.truncate()
2070 self.assertEqual(f.tell(), buffer_size + 2)
2071
2072 @threading_helper.requires_working_threading()
2073 @support.requires_resource('cpu')
2074 def test_threads(self):
2075 try:
2076 # Write out many bytes from many threads and test they were
2077 # all flushed.
2078 N = 1000
2079 contents = bytes(range(256)) * N
2080 sizes = cycle([1, 19])
2081 n = 0
2082 queue = deque()
2083 while n < len(contents):
2084 size = next(sizes)
2085 queue.append(contents[n:n+size])
2086 n += size
2087 del contents
2088 # We use a real file object because it allows us to
2089 # exercise situations where the GIL is released before
2090 # writing the buffer to the raw streams. This is in addition
2091 # to concurrency issues due to switching threads in the middle
2092 # of Python code.
2093 with self.open(os_helper.TESTFN, self.write_mode, buffering=0) as raw:
2094 bufio = self.tp(raw, 8)
2095 errors = []
2096 def f():
2097 try:
2098 while True:
2099 try:
2100 s = queue.popleft()
2101 except IndexError:
2102 return
2103 bufio.write(s)
2104 except Exception as e:
2105 errors.append(e)
2106 raise
2107 threads = [threading.Thread(target=f) for x in range(20)]
2108 with threading_helper.start_threads(threads):
2109 time.sleep(0.02) # yield
2110 self.assertFalse(errors,
2111 "the following exceptions were caught: %r" % errors)
2112 bufio.close()
2113 with self.open(os_helper.TESTFN, "rb") as f:
2114 s = f.read()
2115 for i in range(256):
2116 self.assertEqual(s.count(bytes([i])), N)
2117 finally:
2118 os_helper.unlink(os_helper.TESTFN)
2119
2120 def test_misbehaved_io(self):
2121 rawio = self.MisbehavedRawIO()
2122 bufio = self.tp(rawio, 5)
2123 self.assertRaises(OSError, bufio.seek, 0)
2124 self.assertRaises(OSError, bufio.tell)
2125 self.assertRaises(OSError, bufio.write, b"abcdef")
2126
2127 # Silence destructor error
2128 bufio.close = lambda: None
2129
2130 def test_max_buffer_size_removal(self):
2131 with self.assertRaises(TypeError):
2132 self.tp(self.MockRawIO(), 8, 12)
2133
2134 def test_write_error_on_close(self):
2135 raw = self.MockRawIO()
2136 def bad_write(b):
2137 raise OSError()
2138 raw.write = bad_write
2139 b = self.tp(raw)
2140 b.write(b'spam')
2141 self.assertRaises(OSError, b.close) # exception not swallowed
2142 self.assertTrue(b.closed)
2143
2144 @threading_helper.requires_working_threading()
2145 def test_slow_close_from_thread(self):
2146 # Issue #31976
2147 rawio = self.SlowFlushRawIO()
2148 bufio = self.tp(rawio, 8)
2149 t = threading.Thread(target=bufio.close)
2150 t.start()
2151 rawio.in_flush.wait()
2152 self.assertRaises(ValueError, bufio.write, b'spam')
2153 self.assertTrue(bufio.closed)
2154 t.join()
2155
2156
2157
2158 class CBufferedWriterTest(BufferedWriterTest, SizeofTest):
2159 tp = io.BufferedWriter
2160
2161 def test_initialization(self):
2162 rawio = self.MockRawIO()
2163 bufio = self.tp(rawio)
2164 self.assertRaises(ValueError, bufio.__init__, rawio, buffer_size=0)
2165 self.assertRaises(ValueError, bufio.write, b"def")
2166 self.assertRaises(ValueError, bufio.__init__, rawio, buffer_size=-16)
2167 self.assertRaises(ValueError, bufio.write, b"def")
2168 self.assertRaises(ValueError, bufio.__init__, rawio, buffer_size=-1)
2169 self.assertRaises(ValueError, bufio.write, b"def")
2170
2171 def test_garbage_collection(self):
2172 # C BufferedWriter objects are collected, and collecting them flushes
2173 # all data to disk.
2174 # The Python version has __del__, so it ends into gc.garbage instead
2175 self.addCleanup(os_helper.unlink, os_helper.TESTFN)
2176 with warnings_helper.check_warnings(('', ResourceWarning)):
2177 rawio = self.FileIO(os_helper.TESTFN, "w+b")
2178 f = self.tp(rawio)
2179 f.write(b"123xxx")
2180 f.x = f
2181 wr = weakref.ref(f)
2182 del f
2183 support.gc_collect()
2184 self.assertIsNone(wr(), wr)
2185 with self.open(os_helper.TESTFN, "rb") as f:
2186 self.assertEqual(f.read(), b"123xxx")
2187
2188 def test_args_error(self):
2189 # Issue #17275
2190 with self.assertRaisesRegex(TypeError, "BufferedWriter"):
2191 self.tp(self.BytesIO(), 1024, 1024, 1024)
2192
2193
2194 class PyBufferedWriterTest(BufferedWriterTest):
2195 tp = pyio.BufferedWriter
2196
2197 class BufferedRWPairTest(unittest.TestCase):
2198
2199 def test_constructor(self):
2200 pair = self.tp(self.MockRawIO(), self.MockRawIO())
2201 self.assertFalse(pair.closed)
2202
2203 def test_uninitialized(self):
2204 pair = self.tp.__new__(self.tp)
2205 del pair
2206 pair = self.tp.__new__(self.tp)
2207 self.assertRaisesRegex((ValueError, AttributeError),
2208 'uninitialized|has no attribute',
2209 pair.read, 0)
2210 self.assertRaisesRegex((ValueError, AttributeError),
2211 'uninitialized|has no attribute',
2212 pair.write, b'')
2213 pair.__init__(self.MockRawIO(), self.MockRawIO())
2214 self.assertEqual(pair.read(0), b'')
2215 self.assertEqual(pair.write(b''), 0)
2216
2217 def test_detach(self):
2218 pair = self.tp(self.MockRawIO(), self.MockRawIO())
2219 self.assertRaises(self.UnsupportedOperation, pair.detach)
2220
2221 def test_constructor_max_buffer_size_removal(self):
2222 with self.assertRaises(TypeError):
2223 self.tp(self.MockRawIO(), self.MockRawIO(), 8, 12)
2224
2225 def test_constructor_with_not_readable(self):
2226 class NotReadable(MockRawIO):
2227 def readable(self):
2228 return False
2229
2230 self.assertRaises(OSError, self.tp, NotReadable(), self.MockRawIO())
2231
2232 def test_constructor_with_not_writeable(self):
2233 class NotWriteable(MockRawIO):
2234 def writable(self):
2235 return False
2236
2237 self.assertRaises(OSError, self.tp, self.MockRawIO(), NotWriteable())
2238
2239 def test_read(self):
2240 pair = self.tp(self.BytesIO(b"abcdef"), self.MockRawIO())
2241
2242 self.assertEqual(pair.read(3), b"abc")
2243 self.assertEqual(pair.read(1), b"d")
2244 self.assertEqual(pair.read(), b"ef")
2245 pair = self.tp(self.BytesIO(b"abc"), self.MockRawIO())
2246 self.assertEqual(pair.read(None), b"abc")
2247
2248 def test_readlines(self):
2249 pair = lambda: self.tp(self.BytesIO(b"abc\ndef\nh"), self.MockRawIO())
2250 self.assertEqual(pair().readlines(), [b"abc\n", b"def\n", b"h"])
2251 self.assertEqual(pair().readlines(), [b"abc\n", b"def\n", b"h"])
2252 self.assertEqual(pair().readlines(5), [b"abc\n", b"def\n"])
2253
2254 def test_read1(self):
2255 # .read1() is delegated to the underlying reader object, so this test
2256 # can be shallow.
2257 pair = self.tp(self.BytesIO(b"abcdef"), self.MockRawIO())
2258
2259 self.assertEqual(pair.read1(3), b"abc")
2260 self.assertEqual(pair.read1(), b"def")
2261
2262 def test_readinto(self):
2263 for method in ("readinto", "readinto1"):
2264 with self.subTest(method):
2265 pair = self.tp(self.BytesIO(b"abcdef"), self.MockRawIO())
2266
2267 data = byteslike(b'\0' * 5)
2268 self.assertEqual(getattr(pair, method)(data), 5)
2269 self.assertEqual(bytes(data), b"abcde")
2270
2271 def test_write(self):
2272 w = self.MockRawIO()
2273 pair = self.tp(self.MockRawIO(), w)
2274
2275 pair.write(b"abc")
2276 pair.flush()
2277 buffer = bytearray(b"def")
2278 pair.write(buffer)
2279 buffer[:] = b"***" # Overwrite our copy of the data
2280 pair.flush()
2281 self.assertEqual(w._write_stack, [b"abc", b"def"])
2282
2283 def test_peek(self):
2284 pair = self.tp(self.BytesIO(b"abcdef"), self.MockRawIO())
2285
2286 self.assertStartsWith(pair.peek(3), b"abc")
2287 self.assertEqual(pair.read(3), b"abc")
2288
2289 def test_readable(self):
2290 pair = self.tp(self.MockRawIO(), self.MockRawIO())
2291 self.assertTrue(pair.readable())
2292
2293 def test_writeable(self):
2294 pair = self.tp(self.MockRawIO(), self.MockRawIO())
2295 self.assertTrue(pair.writable())
2296
2297 def test_seekable(self):
2298 # BufferedRWPairs are never seekable, even if their readers and writers
2299 # are.
2300 pair = self.tp(self.MockRawIO(), self.MockRawIO())
2301 self.assertFalse(pair.seekable())
2302
2303 # .flush() is delegated to the underlying writer object and has been
2304 # tested in the test_write method.
2305
2306 def test_close_and_closed(self):
2307 pair = self.tp(self.MockRawIO(), self.MockRawIO())
2308 self.assertFalse(pair.closed)
2309 pair.close()
2310 self.assertTrue(pair.closed)
2311
2312 def test_reader_close_error_on_close(self):
2313 def reader_close():
2314 reader_non_existing
2315 reader = self.MockRawIO()
2316 reader.close = reader_close
2317 writer = self.MockRawIO()
2318 pair = self.tp(reader, writer)
2319 with self.assertRaises(NameError) as err:
2320 pair.close()
2321 self.assertIn('reader_non_existing', str(err.exception))
2322 self.assertTrue(pair.closed)
2323 self.assertFalse(reader.closed)
2324 self.assertTrue(writer.closed)
2325
2326 # Silence destructor error
2327 reader.close = lambda: None
2328
2329 def test_writer_close_error_on_close(self):
2330 def writer_close():
2331 writer_non_existing
2332 reader = self.MockRawIO()
2333 writer = self.MockRawIO()
2334 writer.close = writer_close
2335 pair = self.tp(reader, writer)
2336 with self.assertRaises(NameError) as err:
2337 pair.close()
2338 self.assertIn('writer_non_existing', str(err.exception))
2339 self.assertFalse(pair.closed)
2340 self.assertTrue(reader.closed)
2341 self.assertFalse(writer.closed)
2342
2343 # Silence destructor error
2344 writer.close = lambda: None
2345 writer = None
2346
2347 # Ignore BufferedWriter (of the BufferedRWPair) unraisable exception
2348 with support.catch_unraisable_exception():
2349 # Ignore BufferedRWPair unraisable exception
2350 with support.catch_unraisable_exception():
2351 pair = None
2352 support.gc_collect()
2353 support.gc_collect()
2354
2355 def test_reader_writer_close_error_on_close(self):
2356 def reader_close():
2357 reader_non_existing
2358 def writer_close():
2359 writer_non_existing
2360 reader = self.MockRawIO()
2361 reader.close = reader_close
2362 writer = self.MockRawIO()
2363 writer.close = writer_close
2364 pair = self.tp(reader, writer)
2365 with self.assertRaises(NameError) as err:
2366 pair.close()
2367 self.assertIn('reader_non_existing', str(err.exception))
2368 self.assertIsInstance(err.exception.__context__, NameError)
2369 self.assertIn('writer_non_existing', str(err.exception.__context__))
2370 self.assertFalse(pair.closed)
2371 self.assertFalse(reader.closed)
2372 self.assertFalse(writer.closed)
2373
2374 # Silence destructor error
2375 reader.close = lambda: None
2376 writer.close = lambda: None
2377
2378 def test_isatty(self):
2379 class SelectableIsAtty(MockRawIO):
2380 def __init__(self, isatty):
2381 MockRawIO.__init__(self)
2382 self._isatty = isatty
2383
2384 def isatty(self):
2385 return self._isatty
2386
2387 pair = self.tp(SelectableIsAtty(False), SelectableIsAtty(False))
2388 self.assertFalse(pair.isatty())
2389
2390 pair = self.tp(SelectableIsAtty(True), SelectableIsAtty(False))
2391 self.assertTrue(pair.isatty())
2392
2393 pair = self.tp(SelectableIsAtty(False), SelectableIsAtty(True))
2394 self.assertTrue(pair.isatty())
2395
2396 pair = self.tp(SelectableIsAtty(True), SelectableIsAtty(True))
2397 self.assertTrue(pair.isatty())
2398
2399 def test_weakref_clearing(self):
2400 brw = self.tp(self.MockRawIO(), self.MockRawIO())
2401 ref = weakref.ref(brw)
2402 brw = None
2403 ref = None # Shouldn't segfault.
2404
2405 class CBufferedRWPairTest(BufferedRWPairTest):
2406 tp = io.BufferedRWPair
2407
2408 class PyBufferedRWPairTest(BufferedRWPairTest):
2409 tp = pyio.BufferedRWPair
2410
2411
2412 class BufferedRandomTest(BufferedReaderTest, BufferedWriterTest):
2413 read_mode = "rb+"
2414 write_mode = "wb+"
2415
2416 def test_constructor(self):
2417 BufferedReaderTest.test_constructor(self)
2418 BufferedWriterTest.test_constructor(self)
2419
2420 def test_uninitialized(self):
2421 BufferedReaderTest.test_uninitialized(self)
2422 BufferedWriterTest.test_uninitialized(self)
2423
2424 def test_read_and_write(self):
2425 raw = self.MockRawIO((b"asdf", b"ghjk"))
2426 rw = self.tp(raw, 8)
2427
2428 self.assertEqual(b"as", rw.read(2))
2429 rw.write(b"ddd")
2430 rw.write(b"eee")
2431 self.assertFalse(raw._write_stack) # Buffer writes
2432 self.assertEqual(b"ghjk", rw.read())
2433 self.assertEqual(b"dddeee", raw._write_stack[0])
2434
2435 def test_seek_and_tell(self):
2436 raw = self.BytesIO(b"asdfghjkl")
2437 rw = self.tp(raw)
2438
2439 self.assertEqual(b"as", rw.read(2))
2440 self.assertEqual(2, rw.tell())
2441 rw.seek(0, 0)
2442 self.assertEqual(b"asdf", rw.read(4))
2443
2444 rw.write(b"123f")
2445 rw.seek(0, 0)
2446 self.assertEqual(b"asdf123fl", rw.read())
2447 self.assertEqual(9, rw.tell())
2448 rw.seek(-4, 2)
2449 self.assertEqual(5, rw.tell())
2450 rw.seek(2, 1)
2451 self.assertEqual(7, rw.tell())
2452 self.assertEqual(b"fl", rw.read(11))
2453 rw.flush()
2454 self.assertEqual(b"asdf123fl", raw.getvalue())
2455
2456 self.assertRaises(TypeError, rw.seek, 0.0)
2457
2458 def check_flush_and_read(self, read_func):
2459 raw = self.BytesIO(b"abcdefghi")
2460 bufio = self.tp(raw)
2461
2462 self.assertEqual(b"ab", read_func(bufio, 2))
2463 bufio.write(b"12")
2464 self.assertEqual(b"ef", read_func(bufio, 2))
2465 self.assertEqual(6, bufio.tell())
2466 bufio.flush()
2467 self.assertEqual(6, bufio.tell())
2468 self.assertEqual(b"ghi", read_func(bufio))
2469 raw.seek(0, 0)
2470 raw.write(b"XYZ")
2471 # flush() resets the read buffer
2472 bufio.flush()
2473 bufio.seek(0, 0)
2474 self.assertEqual(b"XYZ", read_func(bufio, 3))
2475
2476 def test_flush_and_read(self):
2477 self.check_flush_and_read(lambda bufio, *args: bufio.read(*args))
2478
2479 def test_flush_and_readinto(self):
2480 def _readinto(bufio, n=-1):
2481 b = bytearray(n if n >= 0 else 9999)
2482 n = bufio.readinto(b)
2483 return bytes(b[:n])
2484 self.check_flush_and_read(_readinto)
2485
2486 def test_flush_and_peek(self):
2487 def _peek(bufio, n=-1):
2488 # This relies on the fact that the buffer can contain the whole
2489 # raw stream, otherwise peek() can return less.
2490 b = bufio.peek(n)
2491 if n != -1:
2492 b = b[:n]
2493 bufio.seek(len(b), 1)
2494 return b
2495 self.check_flush_and_read(_peek)
2496
2497 def test_flush_and_write(self):
2498 raw = self.BytesIO(b"abcdefghi")
2499 bufio = self.tp(raw)
2500
2501 bufio.write(b"123")
2502 bufio.flush()
2503 bufio.write(b"45")
2504 bufio.flush()
2505 bufio.seek(0, 0)
2506 self.assertEqual(b"12345fghi", raw.getvalue())
2507 self.assertEqual(b"12345fghi", bufio.read())
2508
2509 def test_threads(self):
2510 BufferedReaderTest.test_threads(self)
2511 BufferedWriterTest.test_threads(self)
2512
2513 def test_writes_and_peek(self):
2514 def _peek(bufio):
2515 bufio.peek(1)
2516 self.check_writes(_peek)
2517 def _peek(bufio):
2518 pos = bufio.tell()
2519 bufio.seek(-1, 1)
2520 bufio.peek(1)
2521 bufio.seek(pos, 0)
2522 self.check_writes(_peek)
2523
2524 def test_writes_and_reads(self):
2525 def _read(bufio):
2526 bufio.seek(-1, 1)
2527 bufio.read(1)
2528 self.check_writes(_read)
2529
2530 def test_writes_and_read1s(self):
2531 def _read1(bufio):
2532 bufio.seek(-1, 1)
2533 bufio.read1(1)
2534 self.check_writes(_read1)
2535
2536 def test_writes_and_readintos(self):
2537 def _read(bufio):
2538 bufio.seek(-1, 1)
2539 bufio.readinto(bytearray(1))
2540 self.check_writes(_read)
2541
2542 def test_write_after_readahead(self):
2543 # Issue #6629: writing after the buffer was filled by readahead should
2544 # first rewind the raw stream.
2545 for overwrite_size in [1, 5]:
2546 raw = self.BytesIO(b"A" * 10)
2547 bufio = self.tp(raw, 4)
2548 # Trigger readahead
2549 self.assertEqual(bufio.read(1), b"A")
2550 self.assertEqual(bufio.tell(), 1)
2551 # Overwriting should rewind the raw stream if it needs so
2552 bufio.write(b"B" * overwrite_size)
2553 self.assertEqual(bufio.tell(), overwrite_size + 1)
2554 # If the write size was smaller than the buffer size, flush() and
2555 # check that rewind happens.
2556 bufio.flush()
2557 self.assertEqual(bufio.tell(), overwrite_size + 1)
2558 s = raw.getvalue()
2559 self.assertEqual(s,
2560 b"A" + b"B" * overwrite_size + b"A" * (9 - overwrite_size))
2561
2562 def test_write_rewind_write(self):
2563 # Various combinations of reading / writing / seeking backwards / writing again
2564 def mutate(bufio, pos1, pos2):
2565 assert pos2 >= pos1
2566 # Fill the buffer
2567 bufio.seek(pos1)
2568 bufio.read(pos2 - pos1)
2569 bufio.write(b'\x02')
2570 # This writes earlier than the previous write, but still inside
2571 # the buffer.
2572 bufio.seek(pos1)
2573 bufio.write(b'\x01')
2574
2575 b = b"\x80\x81\x82\x83\x84"
2576 for i in range(0, len(b)):
2577 for j in range(i, len(b)):
2578 raw = self.BytesIO(b)
2579 bufio = self.tp(raw, 100)
2580 mutate(bufio, i, j)
2581 bufio.flush()
2582 expected = bytearray(b)
2583 expected[j] = 2
2584 expected[i] = 1
2585 self.assertEqual(raw.getvalue(), expected,
2586 "failed result for i=%d, j=%d" % (i, j))
2587
2588 def test_truncate_after_read_or_write(self):
2589 raw = self.BytesIO(b"A" * 10)
2590 bufio = self.tp(raw, 100)
2591 self.assertEqual(bufio.read(2), b"AA") # the read buffer gets filled
2592 self.assertEqual(bufio.truncate(), 2)
2593 self.assertEqual(bufio.write(b"BB"), 2) # the write buffer increases
2594 self.assertEqual(bufio.truncate(), 4)
2595
2596 def test_misbehaved_io(self):
2597 BufferedReaderTest.test_misbehaved_io(self)
2598 BufferedWriterTest.test_misbehaved_io(self)
2599
2600 def test_interleaved_read_write(self):
2601 # Test for issue #12213
2602 with self.BytesIO(b'abcdefgh') as raw:
2603 with self.tp(raw, 100) as f:
2604 f.write(b"1")
2605 self.assertEqual(f.read(1), b'b')
2606 f.write(b'2')
2607 self.assertEqual(f.read1(1), b'd')
2608 f.write(b'3')
2609 buf = bytearray(1)
2610 f.readinto(buf)
2611 self.assertEqual(buf, b'f')
2612 f.write(b'4')
2613 self.assertEqual(f.peek(1), b'h')
2614 f.flush()
2615 self.assertEqual(raw.getvalue(), b'1b2d3f4h')
2616
2617 with self.BytesIO(b'abc') as raw:
2618 with self.tp(raw, 100) as f:
2619 self.assertEqual(f.read(1), b'a')
2620 f.write(b"2")
2621 self.assertEqual(f.read(1), b'c')
2622 f.flush()
2623 self.assertEqual(raw.getvalue(), b'a2c')
2624
2625 def test_read1_after_write(self):
2626 with self.BytesIO(b'abcdef') as raw:
2627 with self.tp(raw, 3) as f:
2628 f.write(b"1")
2629 self.assertEqual(f.read1(1), b'b')
2630 f.flush()
2631 self.assertEqual(raw.getvalue(), b'1bcdef')
2632 with self.BytesIO(b'abcdef') as raw:
2633 with self.tp(raw, 3) as f:
2634 f.write(b"1")
2635 self.assertEqual(f.read1(), b'bcd')
2636 f.flush()
2637 self.assertEqual(raw.getvalue(), b'1bcdef')
2638 with self.BytesIO(b'abcdef') as raw:
2639 with self.tp(raw, 3) as f:
2640 f.write(b"1")
2641 # XXX: read(100) returns different numbers of bytes
2642 # in Python and C implementations.
2643 self.assertEqual(f.read1(100)[:3], b'bcd')
2644 f.flush()
2645 self.assertEqual(raw.getvalue(), b'1bcdef')
2646
2647 def test_interleaved_readline_write(self):
2648 with self.BytesIO(b'ab\ncdef\ng\n') as raw:
2649 with self.tp(raw) as f:
2650 f.write(b'1')
2651 self.assertEqual(f.readline(), b'b\n')
2652 f.write(b'2')
2653 self.assertEqual(f.readline(), b'def\n')
2654 f.write(b'3')
2655 self.assertEqual(f.readline(), b'\n')
2656 f.flush()
2657 self.assertEqual(raw.getvalue(), b'1b\n2def\n3\n')
2658
2659 # You can't construct a BufferedRandom over a non-seekable stream.
2660 test_unseekable = None
2661
2662 # writable() returns True, so there's no point to test it over
2663 # a writable stream.
2664 test_truncate_on_read_only = None
2665
2666
2667 class CBufferedRandomTest(BufferedRandomTest, SizeofTest):
2668 tp = io.BufferedRandom
2669
2670 def test_garbage_collection(self):
2671 CBufferedReaderTest.test_garbage_collection(self)
2672 CBufferedWriterTest.test_garbage_collection(self)
2673
2674 def test_args_error(self):
2675 # Issue #17275
2676 with self.assertRaisesRegex(TypeError, "BufferedRandom"):
2677 self.tp(self.BytesIO(), 1024, 1024, 1024)
2678
2679
2680 class PyBufferedRandomTest(BufferedRandomTest):
2681 tp = pyio.BufferedRandom
2682
2683
2684 # To fully exercise seek/tell, the StatefulIncrementalDecoder has these
2685 # properties:
2686 # - A single output character can correspond to many bytes of input.
2687 # - The number of input bytes to complete the character can be
2688 # undetermined until the last input byte is received.
2689 # - The number of input bytes can vary depending on previous input.
2690 # - A single input byte can correspond to many characters of output.
2691 # - The number of output characters can be undetermined until the
2692 # last input byte is received.
2693 # - The number of output characters can vary depending on previous input.
2694
2695 class StatefulIncrementalDecoder(codecs.IncrementalDecoder):
2696 """
2697 For testing seek/tell behavior with a stateful, buffering decoder.
2698
2699 Input is a sequence of words. Words may be fixed-length (length set
2700 by input) or variable-length (period-terminated). In variable-length
2701 mode, extra periods are ignored. Possible words are:
2702 - 'i' followed by a number sets the input length, I (maximum 99).
2703 When I is set to 0, words are space-terminated.
2704 - 'o' followed by a number sets the output length, O (maximum 99).
2705 - Any other word is converted into a word followed by a period on
2706 the output. The output word consists of the input word truncated
2707 or padded out with hyphens to make its length equal to O. If O
2708 is 0, the word is output verbatim without truncating or padding.
2709 I and O are initially set to 1. When I changes, any buffered input is
2710 re-scanned according to the new I. EOF also terminates the last word.
2711 """
2712
2713 def __init__(self, errors='strict'):
2714 codecs.IncrementalDecoder.__init__(self, errors)
2715 self.reset()
2716
2717 def __repr__(self):
2718 return '<SID %x>' % id(self)
2719
2720 def reset(self):
2721 self.i = 1
2722 self.o = 1
2723 self.buffer = bytearray()
2724
2725 def getstate(self):
2726 i, o = self.i ^ 1, self.o ^ 1 # so that flags = 0 after reset()
2727 return bytes(self.buffer), i*100 + o
2728
2729 def setstate(self, state):
2730 buffer, io = state
2731 self.buffer = bytearray(buffer)
2732 i, o = divmod(io, 100)
2733 self.i, self.o = i ^ 1, o ^ 1
2734
2735 def decode(self, input, final=False):
2736 output = ''
2737 for b in input:
2738 if self.i == 0: # variable-length, terminated with period
2739 if b == ord('.'):
2740 if self.buffer:
2741 output += self.process_word()
2742 else:
2743 self.buffer.append(b)
2744 else: # fixed-length, terminate after self.i bytes
2745 self.buffer.append(b)
2746 if len(self.buffer) == self.i:
2747 output += self.process_word()
2748 if final and self.buffer: # EOF terminates the last word
2749 output += self.process_word()
2750 return output
2751
2752 def process_word(self):
2753 output = ''
2754 if self.buffer[0] == ord('i'):
2755 self.i = min(99, int(self.buffer[1:] or 0)) # set input length
2756 elif self.buffer[0] == ord('o'):
2757 self.o = min(99, int(self.buffer[1:] or 0)) # set output length
2758 else:
2759 output = self.buffer.decode('ascii')
2760 if len(output) < self.o:
2761 output += '-'*self.o # pad out with hyphens
2762 if self.o:
2763 output = output[:self.o] # truncate to output length
2764 output += '.'
2765 self.buffer = bytearray()
2766 return output
2767
2768 codecEnabled = False
2769
2770
2771 # bpo-41919: This method is separated from StatefulIncrementalDecoder to avoid a resource leak
2772 # when registering codecs and cleanup functions.
2773 def lookupTestDecoder(name):
2774 if StatefulIncrementalDecoder.codecEnabled and name == 'test_decoder':
2775 latin1 = codecs.lookup('latin-1')
2776 return codecs.CodecInfo(
2777 name='test_decoder', encode=latin1.encode, decode=None,
2778 incrementalencoder=None,
2779 streamreader=None, streamwriter=None,
2780 incrementaldecoder=StatefulIncrementalDecoder)
2781
2782
2783 class StatefulIncrementalDecoderTest(unittest.TestCase):
2784 """
2785 Make sure the StatefulIncrementalDecoder actually works.
2786 """
2787
2788 test_cases = [
2789 # I=1, O=1 (fixed-length input == fixed-length output)
2790 (b'abcd', False, 'a.b.c.d.'),
2791 # I=0, O=0 (variable-length input, variable-length output)
2792 (b'oiabcd', True, 'abcd.'),
2793 # I=0, O=0 (should ignore extra periods)
2794 (b'oi...abcd...', True, 'abcd.'),
2795 # I=0, O=6 (variable-length input, fixed-length output)
2796 (b'i.o6.x.xyz.toolongtofit.', False, 'x-----.xyz---.toolon.'),
2797 # I=2, O=6 (fixed-length input < fixed-length output)
2798 (b'i.i2.o6xyz', True, 'xy----.z-----.'),
2799 # I=6, O=3 (fixed-length input > fixed-length output)
2800 (b'i.o3.i6.abcdefghijklmnop', True, 'abc.ghi.mno.'),
2801 # I=0, then 3; O=29, then 15 (with longer output)
2802 (b'i.o29.a.b.cde.o15.abcdefghijabcdefghij.i3.a.b.c.d.ei00k.l.m', True,
2803 'a----------------------------.' +
2804 'b----------------------------.' +
2805 'cde--------------------------.' +
2806 'abcdefghijabcde.' +
2807 'a.b------------.' +
2808 '.c.------------.' +
2809 'd.e------------.' +
2810 'k--------------.' +
2811 'l--------------.' +
2812 'm--------------.')
2813 ]
2814
2815 def test_decoder(self):
2816 # Try a few one-shot test cases.
2817 for input, eof, output in self.test_cases:
2818 d = StatefulIncrementalDecoder()
2819 self.assertEqual(d.decode(input, eof), output)
2820
2821 # Also test an unfinished decode, followed by forcing EOF.
2822 d = StatefulIncrementalDecoder()
2823 self.assertEqual(d.decode(b'oiabcd'), '')
2824 self.assertEqual(d.decode(b'', 1), 'abcd.')
2825
2826 class TextIOWrapperTest(unittest.TestCase):
2827
2828 def setUp(self):
2829 self.testdata = b"AAA\r\nBBB\rCCC\r\nDDD\nEEE\r\n"
2830 self.normalized = b"AAA\nBBB\nCCC\nDDD\nEEE\n".decode("ascii")
2831 os_helper.unlink(os_helper.TESTFN)
2832 codecs.register(lookupTestDecoder)
2833 self.addCleanup(codecs.unregister, lookupTestDecoder)
2834
2835 def tearDown(self):
2836 os_helper.unlink(os_helper.TESTFN)
2837
2838 def test_constructor(self):
2839 r = self.BytesIO(b"\xc3\xa9\n\n")
2840 b = self.BufferedReader(r, 1000)
2841 t = self.TextIOWrapper(b, encoding="utf-8")
2842 t.__init__(b, encoding="latin-1", newline="\r\n")
2843 self.assertEqual(t.encoding, "latin-1")
2844 self.assertEqual(t.line_buffering, False)
2845 t.__init__(b, encoding="utf-8", line_buffering=True)
2846 self.assertEqual(t.encoding, "utf-8")
2847 self.assertEqual(t.line_buffering, True)
2848 self.assertEqual("\xe9\n", t.readline())
2849 invalid_type = TypeError if self.is_C else ValueError
2850 with self.assertRaises(invalid_type):
2851 t.__init__(b, encoding=42)
2852 with self.assertRaises(UnicodeEncodeError):
2853 t.__init__(b, encoding='\udcfe')
2854 with self.assertRaises(ValueError):
2855 t.__init__(b, encoding='utf-8\0')
2856 with self.assertRaises(invalid_type):
2857 t.__init__(b, encoding="utf-8", errors=42)
2858 if support.Py_DEBUG or sys.flags.dev_mode or self.is_C:
2859 with self.assertRaises(UnicodeEncodeError):
2860 t.__init__(b, encoding="utf-8", errors='\udcfe')
2861 if support.Py_DEBUG or sys.flags.dev_mode or self.is_C:
2862 with self.assertRaises(ValueError):
2863 t.__init__(b, encoding="utf-8", errors='replace\0')
2864 with self.assertRaises(TypeError):
2865 t.__init__(b, encoding="utf-8", newline=42)
2866 with self.assertRaises(ValueError):
2867 t.__init__(b, encoding="utf-8", newline='\udcfe')
2868 with self.assertRaises(ValueError):
2869 t.__init__(b, encoding="utf-8", newline='\n\0')
2870 with self.assertRaises(ValueError):
2871 t.__init__(b, encoding="utf-8", newline='xyzzy')
2872
2873 def test_uninitialized(self):
2874 t = self.TextIOWrapper.__new__(self.TextIOWrapper)
2875 del t
2876 t = self.TextIOWrapper.__new__(self.TextIOWrapper)
2877 self.assertRaises(Exception, repr, t)
2878 self.assertRaisesRegex((ValueError, AttributeError),
2879 'uninitialized|has no attribute',
2880 t.read, 0)
2881 t.__init__(self.MockRawIO(), encoding="utf-8")
2882 self.assertEqual(t.read(0), '')
2883
2884 def test_non_text_encoding_codecs_are_rejected(self):
2885 # Ensure the constructor complains if passed a codec that isn't
2886 # marked as a text encoding
2887 # http://bugs.python.org/issue20404
2888 r = self.BytesIO()
2889 b = self.BufferedWriter(r)
2890 with self.assertRaisesRegex(LookupError, "is not a text encoding"):
2891 self.TextIOWrapper(b, encoding="hex")
2892
2893 def test_detach(self):
2894 r = self.BytesIO()
2895 b = self.BufferedWriter(r)
2896 t = self.TextIOWrapper(b, encoding="ascii")
2897 self.assertIs(t.detach(), b)
2898
2899 t = self.TextIOWrapper(b, encoding="ascii")
2900 t.write("howdy")
2901 self.assertFalse(r.getvalue())
2902 t.detach()
2903 self.assertEqual(r.getvalue(), b"howdy")
2904 self.assertRaises(ValueError, t.detach)
2905
2906 # Operations independent of the detached stream should still work
2907 repr(t)
2908 self.assertEqual(t.encoding, "ascii")
2909 self.assertEqual(t.errors, "strict")
2910 self.assertFalse(t.line_buffering)
2911 self.assertFalse(t.write_through)
2912
2913 def test_repr(self):
2914 raw = self.BytesIO("hello".encode("utf-8"))
2915 b = self.BufferedReader(raw)
2916 t = self.TextIOWrapper(b, encoding="utf-8")
2917 modname = self.TextIOWrapper.__module__
2918 self.assertRegex(repr(t),
2919 r"<(%s\.)?TextIOWrapper encoding='utf-8'>" % modname)
2920 raw.name = "dummy"
2921 self.assertRegex(repr(t),
2922 r"<(%s\.)?TextIOWrapper name='dummy' encoding='utf-8'>" % modname)
2923 t.mode = "r"
2924 self.assertRegex(repr(t),
2925 r"<(%s\.)?TextIOWrapper name='dummy' mode='r' encoding='utf-8'>" % modname)
2926 raw.name = b"dummy"
2927 self.assertRegex(repr(t),
2928 r"<(%s\.)?TextIOWrapper name=b'dummy' mode='r' encoding='utf-8'>" % modname)
2929
2930 t.buffer.detach()
2931 repr(t) # Should not raise an exception
2932
2933 def test_recursive_repr(self):
2934 # Issue #25455
2935 raw = self.BytesIO()
2936 t = self.TextIOWrapper(raw, encoding="utf-8")
2937 with support.swap_attr(raw, 'name', t), support.infinite_recursion(25):
2938 with self.assertRaises(RuntimeError):
2939 repr(t) # Should not crash
2940
2941 def test_subclass_repr(self):
2942 class TestSubclass(self.TextIOWrapper):
2943 pass
2944
2945 f = TestSubclass(self.StringIO())
2946 self.assertIn(TestSubclass.__name__, repr(f))
2947
2948 def test_line_buffering(self):
2949 r = self.BytesIO()
2950 b = self.BufferedWriter(r, 1000)
2951 t = self.TextIOWrapper(b, encoding="utf-8", newline="\n", line_buffering=True)
2952 t.write("X")
2953 self.assertEqual(r.getvalue(), b"") # No flush happened
2954 t.write("Y\nZ")
2955 self.assertEqual(r.getvalue(), b"XY\nZ") # All got flushed
2956 t.write("A\rB")
2957 self.assertEqual(r.getvalue(), b"XY\nZA\rB")
2958
2959 def test_reconfigure_line_buffering(self):
2960 r = self.BytesIO()
2961 b = self.BufferedWriter(r, 1000)
2962 t = self.TextIOWrapper(b, encoding="utf-8", newline="\n", line_buffering=False)
2963 t.write("AB\nC")
2964 self.assertEqual(r.getvalue(), b"")
2965
2966 t.reconfigure(line_buffering=True) # implicit flush
2967 self.assertEqual(r.getvalue(), b"AB\nC")
2968 t.write("DEF\nG")
2969 self.assertEqual(r.getvalue(), b"AB\nCDEF\nG")
2970 t.write("H")
2971 self.assertEqual(r.getvalue(), b"AB\nCDEF\nG")
2972 t.reconfigure(line_buffering=False) # implicit flush
2973 self.assertEqual(r.getvalue(), b"AB\nCDEF\nGH")
2974 t.write("IJ")
2975 self.assertEqual(r.getvalue(), b"AB\nCDEF\nGH")
2976
2977 # Keeping default value
2978 t.reconfigure()
2979 t.reconfigure(line_buffering=None)
2980 self.assertEqual(t.line_buffering, False)
2981 t.reconfigure(line_buffering=True)
2982 t.reconfigure()
2983 t.reconfigure(line_buffering=None)
2984 self.assertEqual(t.line_buffering, True)
2985
2986 @unittest.skipIf(sys.flags.utf8_mode, "utf-8 mode is enabled")
2987 def test_default_encoding(self):
2988 with os_helper.EnvironmentVarGuard() as env:
2989 # try to get a user preferred encoding different than the current
2990 # locale encoding to check that TextIOWrapper() uses the current
2991 # locale encoding and not the user preferred encoding
2992 env.unset('LC_ALL', 'LANG', 'LC_CTYPE')
2993
2994 current_locale_encoding = locale.getencoding()
2995 b = self.BytesIO()
2996 with warnings.catch_warnings():
2997 warnings.simplefilter("ignore", EncodingWarning)
2998 t = self.TextIOWrapper(b)
2999 self.assertEqual(t.encoding, current_locale_encoding)
3000
3001 def test_encoding(self):
3002 # Check the encoding attribute is always set, and valid
3003 b = self.BytesIO()
3004 t = self.TextIOWrapper(b, encoding="utf-8")
3005 self.assertEqual(t.encoding, "utf-8")
3006 with warnings.catch_warnings():
3007 warnings.simplefilter("ignore", EncodingWarning)
3008 t = self.TextIOWrapper(b)
3009 self.assertIsNotNone(t.encoding)
3010 codecs.lookup(t.encoding)
3011
3012 def test_encoding_errors_reading(self):
3013 # (1) default
3014 b = self.BytesIO(b"abc\n\xff\n")
3015 t = self.TextIOWrapper(b, encoding="ascii")
3016 self.assertRaises(UnicodeError, t.read)
3017 # (2) explicit strict
3018 b = self.BytesIO(b"abc\n\xff\n")
3019 t = self.TextIOWrapper(b, encoding="ascii", errors="strict")
3020 self.assertRaises(UnicodeError, t.read)
3021 # (3) ignore
3022 b = self.BytesIO(b"abc\n\xff\n")
3023 t = self.TextIOWrapper(b, encoding="ascii", errors="ignore")
3024 self.assertEqual(t.read(), "abc\n\n")
3025 # (4) replace
3026 b = self.BytesIO(b"abc\n\xff\n")
3027 t = self.TextIOWrapper(b, encoding="ascii", errors="replace")
3028 self.assertEqual(t.read(), "abc\n\ufffd\n")
3029
3030 def test_encoding_errors_writing(self):
3031 # (1) default
3032 b = self.BytesIO()
3033 t = self.TextIOWrapper(b, encoding="ascii")
3034 self.assertRaises(UnicodeError, t.write, "\xff")
3035 # (2) explicit strict
3036 b = self.BytesIO()
3037 t = self.TextIOWrapper(b, encoding="ascii", errors="strict")
3038 self.assertRaises(UnicodeError, t.write, "\xff")
3039 # (3) ignore
3040 b = self.BytesIO()
3041 t = self.TextIOWrapper(b, encoding="ascii", errors="ignore",
3042 newline="\n")
3043 t.write("abc\xffdef\n")
3044 t.flush()
3045 self.assertEqual(b.getvalue(), b"abcdef\n")
3046 # (4) replace
3047 b = self.BytesIO()
3048 t = self.TextIOWrapper(b, encoding="ascii", errors="replace",
3049 newline="\n")
3050 t.write("abc\xffdef\n")
3051 t.flush()
3052 self.assertEqual(b.getvalue(), b"abc?def\n")
3053
3054 def test_newlines(self):
3055 input_lines = [ "unix\n", "windows\r\n", "os9\r", "last\n", "nonl" ]
3056
3057 tests = [
3058 [ None, [ 'unix\n', 'windows\n', 'os9\n', 'last\n', 'nonl' ] ],
3059 [ '', input_lines ],
3060 [ '\n', [ "unix\n", "windows\r\n", "os9\rlast\n", "nonl" ] ],
3061 [ '\r\n', [ "unix\nwindows\r\n", "os9\rlast\nnonl" ] ],
3062 [ '\r', [ "unix\nwindows\r", "\nos9\r", "last\nnonl" ] ],
3063 ]
3064 encodings = (
3065 'utf-8', 'latin-1',
3066 'utf-16', 'utf-16-le', 'utf-16-be',
3067 'utf-32', 'utf-32-le', 'utf-32-be',
3068 )
3069
3070 # Try a range of buffer sizes to test the case where \r is the last
3071 # character in TextIOWrapper._pending_line.
3072 for encoding in encodings:
3073 # XXX: str.encode() should return bytes
3074 data = bytes(''.join(input_lines).encode(encoding))
3075 for do_reads in (False, True):
3076 for bufsize in range(1, 10):
3077 for newline, exp_lines in tests:
3078 bufio = self.BufferedReader(self.BytesIO(data), bufsize)
3079 textio = self.TextIOWrapper(bufio, newline=newline,
3080 encoding=encoding)
3081 if do_reads:
3082 got_lines = []
3083 while True:
3084 c2 = textio.read(2)
3085 if c2 == '':
3086 break
3087 self.assertEqual(len(c2), 2)
3088 got_lines.append(c2 + textio.readline())
3089 else:
3090 got_lines = list(textio)
3091
3092 for got_line, exp_line in zip(got_lines, exp_lines):
3093 self.assertEqual(got_line, exp_line)
3094 self.assertEqual(len(got_lines), len(exp_lines))
3095
3096 def test_newlines_input(self):
3097 testdata = b"AAA\nBB\x00B\nCCC\rDDD\rEEE\r\nFFF\r\nGGG"
3098 normalized = testdata.replace(b"\r\n", b"\n").replace(b"\r", b"\n")
3099 for newline, expected in [
3100 (None, normalized.decode("ascii").splitlines(keepends=True)),
3101 ("", testdata.decode("ascii").splitlines(keepends=True)),
3102 ("\n", ["AAA\n", "BB\x00B\n", "CCC\rDDD\rEEE\r\n", "FFF\r\n", "GGG"]),
3103 ("\r\n", ["AAA\nBB\x00B\nCCC\rDDD\rEEE\r\n", "FFF\r\n", "GGG"]),
3104 ("\r", ["AAA\nBB\x00B\nCCC\r", "DDD\r", "EEE\r", "\nFFF\r", "\nGGG"]),
3105 ]:
3106 buf = self.BytesIO(testdata)
3107 txt = self.TextIOWrapper(buf, encoding="ascii", newline=newline)
3108 self.assertEqual(txt.readlines(), expected)
3109 txt.seek(0)
3110 self.assertEqual(txt.read(), "".join(expected))
3111
3112 def test_newlines_output(self):
3113 testdict = {
3114 "": b"AAA\nBBB\nCCC\nX\rY\r\nZ",
3115 "\n": b"AAA\nBBB\nCCC\nX\rY\r\nZ",
3116 "\r": b"AAA\rBBB\rCCC\rX\rY\r\rZ",
3117 "\r\n": b"AAA\r\nBBB\r\nCCC\r\nX\rY\r\r\nZ",
3118 }
3119 tests = [(None, testdict[os.linesep])] + sorted(testdict.items())
3120 for newline, expected in tests:
3121 buf = self.BytesIO()
3122 txt = self.TextIOWrapper(buf, encoding="ascii", newline=newline)
3123 txt.write("AAA\nB")
3124 txt.write("BB\nCCC\n")
3125 txt.write("X\rY\r\nZ")
3126 txt.flush()
3127 self.assertEqual(buf.closed, False)
3128 self.assertEqual(buf.getvalue(), expected)
3129
3130 def test_destructor(self):
3131 l = []
3132 base = self.BytesIO
3133 class MyBytesIO(base):
3134 def close(self):
3135 l.append(self.getvalue())
3136 base.close(self)
3137 b = MyBytesIO()
3138 t = self.TextIOWrapper(b, encoding="ascii")
3139 t.write("abc")
3140 del t
3141 support.gc_collect()
3142 self.assertEqual([b"abc"], l)
3143
3144 def test_override_destructor(self):
3145 record = []
3146 class MyTextIO(self.TextIOWrapper):
3147 def __del__(self):
3148 record.append(1)
3149 try:
3150 f = super().__del__
3151 except AttributeError:
3152 pass
3153 else:
3154 f()
3155 def close(self):
3156 record.append(2)
3157 super().close()
3158 def flush(self):
3159 record.append(3)
3160 super().flush()
3161 b = self.BytesIO()
3162 t = MyTextIO(b, encoding="ascii")
3163 del t
3164 support.gc_collect()
3165 self.assertEqual(record, [1, 2, 3])
3166
3167 def test_error_through_destructor(self):
3168 # Test that the exception state is not modified by a destructor,
3169 # even if close() fails.
3170 rawio = self.CloseFailureIO()
3171 with support.catch_unraisable_exception() as cm:
3172 with self.assertRaises(AttributeError):
3173 self.TextIOWrapper(rawio, encoding="utf-8").xyzzy
3174
3175 self.assertEqual(cm.unraisable.exc_type, OSError)
3176
3177 # Systematic tests of the text I/O API
3178
3179 def test_basic_io(self):
3180 for chunksize in (1, 2, 3, 4, 5, 15, 16, 17, 31, 32, 33, 63, 64, 65):
3181 for enc in "ascii", "latin-1", "utf-8" :# , "utf-16-be", "utf-16-le":
3182 f = self.open(os_helper.TESTFN, "w+", encoding=enc)
3183 f._CHUNK_SIZE = chunksize
3184 self.assertEqual(f.write("abc"), 3)
3185 f.close()
3186 f = self.open(os_helper.TESTFN, "r+", encoding=enc)
3187 f._CHUNK_SIZE = chunksize
3188 self.assertEqual(f.tell(), 0)
3189 self.assertEqual(f.read(), "abc")
3190 cookie = f.tell()
3191 self.assertEqual(f.seek(0), 0)
3192 self.assertEqual(f.read(None), "abc")
3193 f.seek(0)
3194 self.assertEqual(f.read(2), "ab")
3195 self.assertEqual(f.read(1), "c")
3196 self.assertEqual(f.read(1), "")
3197 self.assertEqual(f.read(), "")
3198 self.assertEqual(f.tell(), cookie)
3199 self.assertEqual(f.seek(0), 0)
3200 self.assertEqual(f.seek(0, 2), cookie)
3201 self.assertEqual(f.write("def"), 3)
3202 self.assertEqual(f.seek(cookie), cookie)
3203 self.assertEqual(f.read(), "def")
3204 if enc.startswith("utf"):
3205 self.multi_line_test(f, enc)
3206 f.close()
3207
3208 def multi_line_test(self, f, enc):
3209 f.seek(0)
3210 f.truncate()
3211 sample = "s\xff\u0fff\uffff"
3212 wlines = []
3213 for size in (0, 1, 2, 3, 4, 5, 30, 31, 32, 33, 62, 63, 64, 65, 1000):
3214 chars = []
3215 for i in range(size):
3216 chars.append(sample[i % len(sample)])
3217 line = "".join(chars) + "\n"
3218 wlines.append((f.tell(), line))
3219 f.write(line)
3220 f.seek(0)
3221 rlines = []
3222 while True:
3223 pos = f.tell()
3224 line = f.readline()
3225 if not line:
3226 break
3227 rlines.append((pos, line))
3228 self.assertEqual(rlines, wlines)
3229
3230 def test_telling(self):
3231 f = self.open(os_helper.TESTFN, "w+", encoding="utf-8")
3232 p0 = f.tell()
3233 f.write("\xff\n")
3234 p1 = f.tell()
3235 f.write("\xff\n")
3236 p2 = f.tell()
3237 f.seek(0)
3238 self.assertEqual(f.tell(), p0)
3239 self.assertEqual(f.readline(), "\xff\n")
3240 self.assertEqual(f.tell(), p1)
3241 self.assertEqual(f.readline(), "\xff\n")
3242 self.assertEqual(f.tell(), p2)
3243 f.seek(0)
3244 for line in f:
3245 self.assertEqual(line, "\xff\n")
3246 self.assertRaises(OSError, f.tell)
3247 self.assertEqual(f.tell(), p2)
3248 f.close()
3249
3250 def test_seeking(self):
3251 chunk_size = _default_chunk_size()
3252 prefix_size = chunk_size - 2
3253 u_prefix = "a" * prefix_size
3254 prefix = bytes(u_prefix.encode("utf-8"))
3255 self.assertEqual(len(u_prefix), len(prefix))
3256 u_suffix = "\u8888\n"
3257 suffix = bytes(u_suffix.encode("utf-8"))
3258 line = prefix + suffix
3259 with self.open(os_helper.TESTFN, "wb") as f:
3260 f.write(line*2)
3261 with self.open(os_helper.TESTFN, "r", encoding="utf-8") as f:
3262 s = f.read(prefix_size)
3263 self.assertEqual(s, str(prefix, "ascii"))
3264 self.assertEqual(f.tell(), prefix_size)
3265 self.assertEqual(f.readline(), u_suffix)
3266
3267 def test_seeking_too(self):
3268 # Regression test for a specific bug
3269 data = b'\xe0\xbf\xbf\n'
3270 with self.open(os_helper.TESTFN, "wb") as f:
3271 f.write(data)
3272 with self.open(os_helper.TESTFN, "r", encoding="utf-8") as f:
3273 f._CHUNK_SIZE # Just test that it exists
3274 f._CHUNK_SIZE = 2
3275 f.readline()
3276 f.tell()
3277
3278 def test_seek_and_tell(self):
3279 #Test seek/tell using the StatefulIncrementalDecoder.
3280 # Make test faster by doing smaller seeks
3281 CHUNK_SIZE = 128
3282
3283 def test_seek_and_tell_with_data(data, min_pos=0):
3284 """Tell/seek to various points within a data stream and ensure
3285 that the decoded data returned by read() is consistent."""
3286 f = self.open(os_helper.TESTFN, 'wb')
3287 f.write(data)
3288 f.close()
3289 f = self.open(os_helper.TESTFN, encoding='test_decoder')
3290 f._CHUNK_SIZE = CHUNK_SIZE
3291 decoded = f.read()
3292 f.close()
3293
3294 for i in range(min_pos, len(decoded) + 1): # seek positions
3295 for j in [1, 5, len(decoded) - i]: # read lengths
3296 f = self.open(os_helper.TESTFN, encoding='test_decoder')
3297 self.assertEqual(f.read(i), decoded[:i])
3298 cookie = f.tell()
3299 self.assertEqual(f.read(j), decoded[i:i + j])
3300 f.seek(cookie)
3301 self.assertEqual(f.read(), decoded[i:])
3302 f.close()
3303
3304 # Enable the test decoder.
3305 StatefulIncrementalDecoder.codecEnabled = 1
3306
3307 # Run the tests.
3308 try:
3309 # Try each test case.
3310 for input, _, _ in StatefulIncrementalDecoderTest.test_cases:
3311 test_seek_and_tell_with_data(input)
3312
3313 # Position each test case so that it crosses a chunk boundary.
3314 for input, _, _ in StatefulIncrementalDecoderTest.test_cases:
3315 offset = CHUNK_SIZE - len(input)//2
3316 prefix = b'.'*offset
3317 # Don't bother seeking into the prefix (takes too long).
3318 min_pos = offset*2
3319 test_seek_and_tell_with_data(prefix + input, min_pos)
3320
3321 # Ensure our test decoder won't interfere with subsequent tests.
3322 finally:
3323 StatefulIncrementalDecoder.codecEnabled = 0
3324
3325 def test_multibyte_seek_and_tell(self):
3326 f = self.open(os_helper.TESTFN, "w", encoding="euc_jp")
3327 f.write("AB\n\u3046\u3048\n")
3328 f.close()
3329
3330 f = self.open(os_helper.TESTFN, "r", encoding="euc_jp")
3331 self.assertEqual(f.readline(), "AB\n")
3332 p0 = f.tell()
3333 self.assertEqual(f.readline(), "\u3046\u3048\n")
3334 p1 = f.tell()
3335 f.seek(p0)
3336 self.assertEqual(f.readline(), "\u3046\u3048\n")
3337 self.assertEqual(f.tell(), p1)
3338 f.close()
3339
3340 def test_tell_after_readline_with_cr(self):
3341 # Test for gh-141314: TextIOWrapper.tell() assertion failure
3342 # when dealing with standalone carriage returns
3343 data = b'line1\r'
3344 with self.open(os_helper.TESTFN, "wb") as f:
3345 f.write(data)
3346
3347 with self.open(os_helper.TESTFN, "r") as f:
3348 # Read line that ends with \r
3349 line = f.readline()
3350 self.assertEqual(line, "line1\n")
3351 # This should not cause an assertion failure
3352 pos = f.tell()
3353 # Verify we can seek back to this position
3354 f.seek(pos)
3355 remaining = f.read()
3356 self.assertEqual(remaining, "")
3357
3358 def test_seek_with_encoder_state(self):
3359 f = self.open(os_helper.TESTFN, "w", encoding="euc_jis_2004")
3360 f.write("\u00e6\u0300")
3361 p0 = f.tell()
3362 f.write("\u00e6")
3363 f.seek(p0)
3364 f.write("\u0300")
3365 f.close()
3366
3367 f = self.open(os_helper.TESTFN, "r", encoding="euc_jis_2004")
3368 self.assertEqual(f.readline(), "\u00e6\u0300\u0300")
3369 f.close()
3370
3371 def test_encoded_writes(self):
3372 data = "1234567890"
3373 tests = ("utf-16",
3374 "utf-16-le",
3375 "utf-16-be",
3376 "utf-32",
3377 "utf-32-le",
3378 "utf-32-be")
3379 for encoding in tests:
3380 buf = self.BytesIO()
3381 f = self.TextIOWrapper(buf, encoding=encoding)
3382 # Check if the BOM is written only once (see issue1753).
3383 f.write(data)
3384 f.write(data)
3385 f.seek(0)
3386 self.assertEqual(f.read(), data * 2)
3387 f.seek(0)
3388 self.assertEqual(f.read(), data * 2)
3389 self.assertEqual(buf.getvalue(), (data * 2).encode(encoding))
3390
3391 def test_unreadable(self):
3392 class UnReadable(self.BytesIO):
3393 def readable(self):
3394 return False
3395 txt = self.TextIOWrapper(UnReadable(), encoding="utf-8")
3396 self.assertRaises(OSError, txt.read)
3397
3398 def test_read_one_by_one(self):
3399 txt = self.TextIOWrapper(self.BytesIO(b"AA\r\nBB"), encoding="utf-8")
3400 reads = ""
3401 while True:
3402 c = txt.read(1)
3403 if not c:
3404 break
3405 reads += c
3406 self.assertEqual(reads, "AA\nBB")
3407
3408 def test_readlines(self):
3409 txt = self.TextIOWrapper(self.BytesIO(b"AA\nBB\nCC"), encoding="utf-8")
3410 self.assertEqual(txt.readlines(), ["AA\n", "BB\n", "CC"])
3411 txt.seek(0)
3412 self.assertEqual(txt.readlines(None), ["AA\n", "BB\n", "CC"])
3413 txt.seek(0)
3414 self.assertEqual(txt.readlines(5), ["AA\n", "BB\n"])
3415
3416 # read in amounts equal to TextIOWrapper._CHUNK_SIZE which is 128.
3417 def test_read_by_chunk(self):
3418 # make sure "\r\n" straddles 128 char boundary.
3419 txt = self.TextIOWrapper(self.BytesIO(b"A" * 127 + b"\r\nB"), encoding="utf-8")
3420 reads = ""
3421 while True:
3422 c = txt.read(128)
3423 if not c:
3424 break
3425 reads += c
3426 self.assertEqual(reads, "A"*127+"\nB")
3427
3428 def test_writelines(self):
3429 l = ['ab', 'cd', 'ef']
3430 buf = self.BytesIO()
3431 txt = self.TextIOWrapper(buf, encoding="utf-8")
3432 txt.writelines(l)
3433 txt.flush()
3434 self.assertEqual(buf.getvalue(), b'abcdef')
3435
3436 def test_writelines_userlist(self):
3437 l = UserList(['ab', 'cd', 'ef'])
3438 buf = self.BytesIO()
3439 txt = self.TextIOWrapper(buf, encoding="utf-8")
3440 txt.writelines(l)
3441 txt.flush()
3442 self.assertEqual(buf.getvalue(), b'abcdef')
3443
3444 def test_writelines_error(self):
3445 txt = self.TextIOWrapper(self.BytesIO(), encoding="utf-8")
3446 self.assertRaises(TypeError, txt.writelines, [1, 2, 3])
3447 self.assertRaises(TypeError, txt.writelines, None)
3448 self.assertRaises(TypeError, txt.writelines, b'abc')
3449
3450 def test_issue1395_1(self):
3451 txt = self.TextIOWrapper(self.BytesIO(self.testdata), encoding="ascii")
3452
3453 # read one char at a time
3454 reads = ""
3455 while True:
3456 c = txt.read(1)
3457 if not c:
3458 break
3459 reads += c
3460 self.assertEqual(reads, self.normalized)
3461
3462 def test_issue1395_2(self):
3463 txt = self.TextIOWrapper(self.BytesIO(self.testdata), encoding="ascii")
3464 txt._CHUNK_SIZE = 4
3465
3466 reads = ""
3467 while True:
3468 c = txt.read(4)
3469 if not c:
3470 break
3471 reads += c
3472 self.assertEqual(reads, self.normalized)
3473
3474 def test_issue1395_3(self):
3475 txt = self.TextIOWrapper(self.BytesIO(self.testdata), encoding="ascii")
3476 txt._CHUNK_SIZE = 4
3477
3478 reads = txt.read(4)
3479 reads += txt.read(4)
3480 reads += txt.readline()
3481 reads += txt.readline()
3482 reads += txt.readline()
3483 self.assertEqual(reads, self.normalized)
3484
3485 def test_issue1395_4(self):
3486 txt = self.TextIOWrapper(self.BytesIO(self.testdata), encoding="ascii")
3487 txt._CHUNK_SIZE = 4
3488
3489 reads = txt.read(4)
3490 reads += txt.read()
3491 self.assertEqual(reads, self.normalized)
3492
3493 def test_issue1395_5(self):
3494 txt = self.TextIOWrapper(self.BytesIO(self.testdata), encoding="ascii")
3495 txt._CHUNK_SIZE = 4
3496
3497 reads = txt.read(4)
3498 pos = txt.tell()
3499 txt.seek(0)
3500 txt.seek(pos)
3501 self.assertEqual(txt.read(4), "BBB\n")
3502
3503 def test_issue2282(self):
3504 buffer = self.BytesIO(self.testdata)
3505 txt = self.TextIOWrapper(buffer, encoding="ascii")
3506
3507 self.assertEqual(buffer.seekable(), txt.seekable())
3508
3509 def test_append_bom(self):
3510 # The BOM is not written again when appending to a non-empty file
3511 filename = os_helper.TESTFN
3512 for charset in ('utf-8-sig', 'utf-16', 'utf-32'):
3513 with self.open(filename, 'w', encoding=charset) as f:
3514 f.write('aaa')
3515 pos = f.tell()
3516 with self.open(filename, 'rb') as f:
3517 self.assertEqual(f.read(), 'aaa'.encode(charset))
3518
3519 with self.open(filename, 'a', encoding=charset) as f:
3520 f.write('xxx')
3521 with self.open(filename, 'rb') as f:
3522 self.assertEqual(f.read(), 'aaaxxx'.encode(charset))
3523
3524 def test_seek_bom(self):
3525 # Same test, but when seeking manually
3526 filename = os_helper.TESTFN
3527 for charset in ('utf-8-sig', 'utf-16', 'utf-32'):
3528 with self.open(filename, 'w', encoding=charset) as f:
3529 f.write('aaa')
3530 pos = f.tell()
3531 with self.open(filename, 'r+', encoding=charset) as f:
3532 f.seek(pos)
3533 f.write('zzz')
3534 f.seek(0)
3535 f.write('bbb')
3536 with self.open(filename, 'rb') as f:
3537 self.assertEqual(f.read(), 'bbbzzz'.encode(charset))
3538
3539 def test_seek_append_bom(self):
3540 # Same test, but first seek to the start and then to the end
3541 filename = os_helper.TESTFN
3542 for charset in ('utf-8-sig', 'utf-16', 'utf-32'):
3543 with self.open(filename, 'w', encoding=charset) as f:
3544 f.write('aaa')
3545 with self.open(filename, 'a', encoding=charset) as f:
3546 f.seek(0)
3547 f.seek(0, self.SEEK_END)
3548 f.write('xxx')
3549 with self.open(filename, 'rb') as f:
3550 self.assertEqual(f.read(), 'aaaxxx'.encode(charset))
3551
3552 def test_errors_property(self):
3553 with self.open(os_helper.TESTFN, "w", encoding="utf-8") as f:
3554 self.assertEqual(f.errors, "strict")
3555 with self.open(os_helper.TESTFN, "w", encoding="utf-8", errors="replace") as f:
3556 self.assertEqual(f.errors, "replace")
3557
3558 @support.no_tracing
3559 @threading_helper.requires_working_threading()
3560 def test_threads_write(self):
3561 # Issue6750: concurrent writes could duplicate data
3562 event = threading.Event()
3563 with self.open(os_helper.TESTFN, "w", encoding="utf-8", buffering=1) as f:
3564 def run(n):
3565 text = "Thread%03d\n" % n
3566 event.wait()
3567 f.write(text)
3568 threads = [threading.Thread(target=run, args=(x,))
3569 for x in range(20)]
3570 with threading_helper.start_threads(threads, event.set):
3571 time.sleep(0.02)
3572 with self.open(os_helper.TESTFN, encoding="utf-8") as f:
3573 content = f.read()
3574 for n in range(20):
3575 self.assertEqual(content.count("Thread%03d\n" % n), 1)
3576
3577 def test_flush_error_on_close(self):
3578 # Test that text file is closed despite failed flush
3579 # and that flush() is called before file closed.
3580 txt = self.TextIOWrapper(self.BytesIO(self.testdata), encoding="ascii")
3581 closed = []
3582 def bad_flush():
3583 closed[:] = [txt.closed, txt.buffer.closed]
3584 raise OSError()
3585 txt.flush = bad_flush
3586 self.assertRaises(OSError, txt.close) # exception not swallowed
3587 self.assertTrue(txt.closed)
3588 self.assertTrue(txt.buffer.closed)
3589 self.assertTrue(closed) # flush() called
3590 self.assertFalse(closed[0]) # flush() called before file closed
3591 self.assertFalse(closed[1])
3592 txt.flush = lambda: None # break reference loop
3593
3594 def test_close_error_on_close(self):
3595 buffer = self.BytesIO(self.testdata)
3596 def bad_flush():
3597 raise OSError('flush')
3598 def bad_close():
3599 raise OSError('close')
3600 buffer.close = bad_close
3601 txt = self.TextIOWrapper(buffer, encoding="ascii")
3602 txt.flush = bad_flush
3603 with self.assertRaises(OSError) as err: # exception not swallowed
3604 txt.close()
3605 self.assertEqual(err.exception.args, ('close',))
3606 self.assertIsInstance(err.exception.__context__, OSError)
3607 self.assertEqual(err.exception.__context__.args, ('flush',))
3608 self.assertFalse(txt.closed)
3609
3610 # Silence destructor error
3611 buffer.close = lambda: None
3612 txt.flush = lambda: None
3613
3614 def test_nonnormalized_close_error_on_close(self):
3615 # Issue #21677
3616 buffer = self.BytesIO(self.testdata)
3617 def bad_flush():
3618 raise non_existing_flush
3619 def bad_close():
3620 raise non_existing_close
3621 buffer.close = bad_close
3622 txt = self.TextIOWrapper(buffer, encoding="ascii")
3623 txt.flush = bad_flush
3624 with self.assertRaises(NameError) as err: # exception not swallowed
3625 txt.close()
3626 self.assertIn('non_existing_close', str(err.exception))
3627 self.assertIsInstance(err.exception.__context__, NameError)
3628 self.assertIn('non_existing_flush', str(err.exception.__context__))
3629 self.assertFalse(txt.closed)
3630
3631 # Silence destructor error
3632 buffer.close = lambda: None
3633 txt.flush = lambda: None
3634
3635 def test_multi_close(self):
3636 txt = self.TextIOWrapper(self.BytesIO(self.testdata), encoding="ascii")
3637 txt.close()
3638 txt.close()
3639 txt.close()
3640 self.assertRaises(ValueError, txt.flush)
3641
3642 def test_unseekable(self):
3643 txt = self.TextIOWrapper(self.MockUnseekableIO(self.testdata), encoding="utf-8")
3644 self.assertRaises(self.UnsupportedOperation, txt.tell)
3645 self.assertRaises(self.UnsupportedOperation, txt.seek, 0)
3646
3647 def test_readonly_attributes(self):
3648 txt = self.TextIOWrapper(self.BytesIO(self.testdata), encoding="ascii")
3649 buf = self.BytesIO(self.testdata)
3650 with self.assertRaises(AttributeError):
3651 txt.buffer = buf
3652
3653 def test_rawio(self):
3654 # Issue #12591: TextIOWrapper must work with raw I/O objects, so
3655 # that subprocess.Popen() can have the required unbuffered
3656 # semantics with universal_newlines=True.
3657 raw = self.MockRawIO([b'abc', b'def', b'ghi\njkl\nopq\n'])
3658 txt = self.TextIOWrapper(raw, encoding='ascii', newline='\n')
3659 # Reads
3660 self.assertEqual(txt.read(4), 'abcd')
3661 self.assertEqual(txt.readline(), 'efghi\n')
3662 self.assertEqual(list(txt), ['jkl\n', 'opq\n'])
3663
3664 def test_rawio_write_through(self):
3665 # Issue #12591: with write_through=True, writes don't need a flush
3666 raw = self.MockRawIO([b'abc', b'def', b'ghi\njkl\nopq\n'])
3667 txt = self.TextIOWrapper(raw, encoding='ascii', newline='\n',
3668 write_through=True)
3669 txt.write('1')
3670 txt.write('23\n4')
3671 txt.write('5')
3672 self.assertEqual(b''.join(raw._write_stack), b'123\n45')
3673
3674 def test_bufio_write_through(self):
3675 # Issue #21396: write_through=True doesn't force a flush()
3676 # on the underlying binary buffered object.
3677 flush_called, write_called = [], []
3678 class BufferedWriter(self.BufferedWriter):
3679 def flush(self, *args, **kwargs):
3680 flush_called.append(True)
3681 return super().flush(*args, **kwargs)
3682 def write(self, *args, **kwargs):
3683 write_called.append(True)
3684 return super().write(*args, **kwargs)
3685
3686 rawio = self.BytesIO()
3687 data = b"a"
3688 bufio = BufferedWriter(rawio, len(data)*2)
3689 textio = self.TextIOWrapper(bufio, encoding='ascii',
3690 write_through=True)
3691 # write to the buffered io but don't overflow the buffer
3692 text = data.decode('ascii')
3693 textio.write(text)
3694
3695 # buffer.flush is not called with write_through=True
3696 self.assertFalse(flush_called)
3697 # buffer.write *is* called with write_through=True
3698 self.assertTrue(write_called)
3699 self.assertEqual(rawio.getvalue(), b"") # no flush
3700
3701 write_called = [] # reset
3702 textio.write(text * 10) # total content is larger than bufio buffer
3703 self.assertTrue(write_called)
3704 self.assertEqual(rawio.getvalue(), data * 11) # all flushed
3705
3706 def test_reconfigure_write_through(self):
3707 raw = self.MockRawIO([])
3708 t = self.TextIOWrapper(raw, encoding='ascii', newline='\n')
3709 t.write('1')
3710 t.reconfigure(write_through=True) # implied flush
3711 self.assertEqual(t.write_through, True)
3712 self.assertEqual(b''.join(raw._write_stack), b'1')
3713 t.write('23')
3714 self.assertEqual(b''.join(raw._write_stack), b'123')
3715 t.reconfigure(write_through=False)
3716 self.assertEqual(t.write_through, False)
3717 t.write('45')
3718 t.flush()
3719 self.assertEqual(b''.join(raw._write_stack), b'12345')
3720 # Keeping default value
3721 t.reconfigure()
3722 t.reconfigure(write_through=None)
3723 self.assertEqual(t.write_through, False)
3724 t.reconfigure(write_through=True)
3725 t.reconfigure()
3726 t.reconfigure(write_through=None)
3727 self.assertEqual(t.write_through, True)
3728
3729 def test_read_nonbytes(self):
3730 # Issue #17106
3731 # Crash when underlying read() returns non-bytes
3732 t = self.TextIOWrapper(self.StringIO('a'), encoding="utf-8")
3733 self.assertRaises(TypeError, t.read, 1)
3734 t = self.TextIOWrapper(self.StringIO('a'), encoding="utf-8")
3735 self.assertRaises(TypeError, t.readline)
3736 t = self.TextIOWrapper(self.StringIO('a'), encoding="utf-8")
3737 self.assertRaises(TypeError, t.read)
3738
3739 def test_illegal_encoder(self):
3740 # Issue 31271: Calling write() while the return value of encoder's
3741 # encode() is invalid shouldn't cause an assertion failure.
3742 rot13 = codecs.lookup("rot13")
3743 with support.swap_attr(rot13, '_is_text_encoding', True):
3744 t = self.TextIOWrapper(self.BytesIO(b'foo'), encoding="rot13")
3745 self.assertRaises(TypeError, t.write, 'bar')
3746
3747 def test_illegal_decoder(self):
3748 # Issue #17106
3749 # Bypass the early encoding check added in issue 20404
3750 def _make_illegal_wrapper():
3751 quopri = codecs.lookup("quopri")
3752 quopri._is_text_encoding = True
3753 try:
3754 t = self.TextIOWrapper(self.BytesIO(b'aaaaaa'),
3755 newline='\n', encoding="quopri")
3756 finally:
3757 quopri._is_text_encoding = False
3758 return t
3759 # Crash when decoder returns non-string
3760 t = _make_illegal_wrapper()
3761 self.assertRaises(TypeError, t.read, 1)
3762 t = _make_illegal_wrapper()
3763 self.assertRaises(TypeError, t.readline)
3764 t = _make_illegal_wrapper()
3765 self.assertRaises(TypeError, t.read)
3766
3767 # Issue 31243: calling read() while the return value of decoder's
3768 # getstate() is invalid should neither crash the interpreter nor
3769 # raise a SystemError.
3770 def _make_very_illegal_wrapper(getstate_ret_val):
3771 class BadDecoder:
3772 def getstate(self):
3773 return getstate_ret_val
3774 def _get_bad_decoder(dummy):
3775 return BadDecoder()
3776 quopri = codecs.lookup("quopri")
3777 with support.swap_attr(quopri, 'incrementaldecoder',
3778 _get_bad_decoder):
3779 return _make_illegal_wrapper()
3780 t = _make_very_illegal_wrapper(42)
3781 self.assertRaises(TypeError, t.read, 42)
3782 t = _make_very_illegal_wrapper(())
3783 self.assertRaises(TypeError, t.read, 42)
3784 t = _make_very_illegal_wrapper((1, 2))
3785 self.assertRaises(TypeError, t.read, 42)
3786
3787 def _check_create_at_shutdown(self, **kwargs):
3788 # Issue #20037: creating a TextIOWrapper at shutdown
3789 # shouldn't crash the interpreter.
3790 iomod = self.io.__name__
3791 code = """if 1:
3792 import codecs
3793 import {iomod} as io
3794
3795 # Avoid looking up codecs at shutdown
3796 codecs.lookup('utf-8')
3797
3798 class C:
3799 def __del__(self):
3800 io.TextIOWrapper(io.BytesIO(), **{kwargs})
3801 print("ok")
3802 c = C()
3803 """.format(iomod=iomod, kwargs=kwargs)
3804 return assert_python_ok("-c", code)
3805
3806 def test_create_at_shutdown_without_encoding(self):
3807 rc, out, err = self._check_create_at_shutdown()
3808 if err:
3809 # Can error out with a RuntimeError if the module state
3810 # isn't found.
3811 self.assertIn(self.shutdown_error, err.decode())
3812 else:
3813 self.assertEqual("ok", out.decode().strip())
3814
3815 def test_create_at_shutdown_with_encoding(self):
3816 rc, out, err = self._check_create_at_shutdown(encoding='utf-8',
3817 errors='strict')
3818 self.assertFalse(err)
3819 self.assertEqual("ok", out.decode().strip())
3820
3821 def test_read_byteslike(self):
3822 r = MemviewBytesIO(b'Just some random string\n')
3823 t = self.TextIOWrapper(r, 'utf-8')
3824
3825 # TextIOwrapper will not read the full string, because
3826 # we truncate it to a multiple of the native int size
3827 # so that we can construct a more complex memoryview.
3828 bytes_val = _to_memoryview(r.getvalue()).tobytes()
3829
3830 self.assertEqual(t.read(200), bytes_val.decode('utf-8'))
3831
3832 def test_issue22849(self):
3833 class F(object):
3834 def readable(self): return True
3835 def writable(self): return True
3836 def seekable(self): return True
3837
3838 for i in range(10):
3839 try:
3840 self.TextIOWrapper(F(), encoding='utf-8')
3841 except Exception:
3842 pass
3843
3844 F.tell = lambda x: 0
3845 t = self.TextIOWrapper(F(), encoding='utf-8')
3846
3847 def test_reconfigure_locale(self):
3848 wrapper = self.TextIOWrapper(self.BytesIO(b"test"))
3849 wrapper.reconfigure(encoding="locale")
3850
3851 def test_reconfigure_encoding_read(self):
3852 # latin1 -> utf8
3853 # (latin1 can decode utf-8 encoded string)
3854 data = 'abc\xe9\n'.encode('latin1') + 'd\xe9f\n'.encode('utf8')
3855 raw = self.BytesIO(data)
3856 txt = self.TextIOWrapper(raw, encoding='latin1', newline='\n')
3857 self.assertEqual(txt.readline(), 'abc\xe9\n')
3858 with self.assertRaises(self.UnsupportedOperation):
3859 txt.reconfigure(encoding='utf-8')
3860 with self.assertRaises(self.UnsupportedOperation):
3861 txt.reconfigure(newline=None)
3862
3863 def test_reconfigure_write_fromascii(self):
3864 # ascii has a specific encodefunc in the C implementation,
3865 # but utf-8-sig has not. Make sure that we get rid of the
3866 # cached encodefunc when we switch encoders.
3867 raw = self.BytesIO()
3868 txt = self.TextIOWrapper(raw, encoding='ascii', newline='\n')
3869 txt.write('foo\n')
3870 txt.reconfigure(encoding='utf-8-sig')
3871 txt.write('\xe9\n')
3872 txt.flush()
3873 self.assertEqual(raw.getvalue(), b'foo\n\xc3\xa9\n')
3874
3875 def test_reconfigure_write(self):
3876 # latin -> utf8
3877 raw = self.BytesIO()
3878 txt = self.TextIOWrapper(raw, encoding='latin1', newline='\n')
3879 txt.write('abc\xe9\n')
3880 txt.reconfigure(encoding='utf-8')
3881 self.assertEqual(raw.getvalue(), b'abc\xe9\n')
3882 txt.write('d\xe9f\n')
3883 txt.flush()
3884 self.assertEqual(raw.getvalue(), b'abc\xe9\nd\xc3\xa9f\n')
3885
3886 # ascii -> utf-8-sig: ensure that no BOM is written in the middle of
3887 # the file
3888 raw = self.BytesIO()
3889 txt = self.TextIOWrapper(raw, encoding='ascii', newline='\n')
3890 txt.write('abc\n')
3891 txt.reconfigure(encoding='utf-8-sig')
3892 txt.write('d\xe9f\n')
3893 txt.flush()
3894 self.assertEqual(raw.getvalue(), b'abc\nd\xc3\xa9f\n')
3895
3896 def test_reconfigure_write_non_seekable(self):
3897 raw = self.BytesIO()
3898 raw.seekable = lambda: False
3899 raw.seek = None
3900 txt = self.TextIOWrapper(raw, encoding='ascii', newline='\n')
3901 txt.write('abc\n')
3902 txt.reconfigure(encoding='utf-8-sig')
3903 txt.write('d\xe9f\n')
3904 txt.flush()
3905
3906 # If the raw stream is not seekable, there'll be a BOM
3907 self.assertEqual(raw.getvalue(), b'abc\n\xef\xbb\xbfd\xc3\xa9f\n')
3908
3909 def test_reconfigure_defaults(self):
3910 txt = self.TextIOWrapper(self.BytesIO(), 'ascii', 'replace', '\n')
3911 txt.reconfigure(encoding=None)
3912 self.assertEqual(txt.encoding, 'ascii')
3913 self.assertEqual(txt.errors, 'replace')
3914 txt.write('LF\n')
3915
3916 txt.reconfigure(newline='\r\n')
3917 self.assertEqual(txt.encoding, 'ascii')
3918 self.assertEqual(txt.errors, 'replace')
3919
3920 txt.reconfigure(errors='ignore')
3921 self.assertEqual(txt.encoding, 'ascii')
3922 self.assertEqual(txt.errors, 'ignore')
3923 txt.write('CRLF\n')
3924
3925 txt.reconfigure(encoding='utf-8', newline=None)
3926 self.assertEqual(txt.errors, 'strict')
3927 txt.seek(0)
3928 self.assertEqual(txt.read(), 'LF\nCRLF\n')
3929
3930 self.assertEqual(txt.detach().getvalue(), b'LF\nCRLF\r\n')
3931
3932 def test_reconfigure_errors(self):
3933 txt = self.TextIOWrapper(self.BytesIO(), 'ascii', 'replace', '\r')
3934 with self.assertRaises(TypeError): # there was a crash
3935 txt.reconfigure(encoding=42)
3936 if self.is_C:
3937 with self.assertRaises(UnicodeEncodeError):
3938 txt.reconfigure(encoding='\udcfe')
3939 with self.assertRaises(LookupError):
3940 txt.reconfigure(encoding='locale\0')
3941 # TODO: txt.reconfigure(encoding='utf-8\0')
3942 # TODO: txt.reconfigure(encoding='nonexisting')
3943 with self.assertRaises(TypeError):
3944 txt.reconfigure(errors=42)
3945 if self.is_C:
3946 with self.assertRaises(UnicodeEncodeError):
3947 txt.reconfigure(errors='\udcfe')
3948 # TODO: txt.reconfigure(errors='ignore\0')
3949 # TODO: txt.reconfigure(errors='nonexisting')
3950 with self.assertRaises(TypeError):
3951 txt.reconfigure(newline=42)
3952 with self.assertRaises(ValueError):
3953 txt.reconfigure(newline='\udcfe')
3954 with self.assertRaises(ValueError):
3955 txt.reconfigure(newline='xyz')
3956 if not self.is_C:
3957 # TODO: Should fail in C too.
3958 with self.assertRaises(ValueError):
3959 txt.reconfigure(newline='\n\0')
3960 if self.is_C:
3961 # TODO: Use __bool__(), not __index__().
3962 with self.assertRaises(ZeroDivisionError):
3963 txt.reconfigure(line_buffering=BadIndex())
3964 with self.assertRaises(OverflowError):
3965 txt.reconfigure(line_buffering=2**1000)
3966 with self.assertRaises(ZeroDivisionError):
3967 txt.reconfigure(write_through=BadIndex())
3968 with self.assertRaises(OverflowError):
3969 txt.reconfigure(write_through=2**1000)
3970 with self.assertRaises(ZeroDivisionError): # there was a crash
3971 txt.reconfigure(line_buffering=BadIndex(),
3972 write_through=BadIndex())
3973 self.assertEqual(txt.encoding, 'ascii')
3974 self.assertEqual(txt.errors, 'replace')
3975 self.assertIs(txt.line_buffering, False)
3976 self.assertIs(txt.write_through, False)
3977
3978 txt.reconfigure(encoding='latin1', errors='ignore', newline='\r\n',
3979 line_buffering=True, write_through=True)
3980 self.assertEqual(txt.encoding, 'latin1')
3981 self.assertEqual(txt.errors, 'ignore')
3982 self.assertIs(txt.line_buffering, True)
3983 self.assertIs(txt.write_through, True)
3984
3985 def test_reconfigure_newline(self):
3986 raw = self.BytesIO(b'CR\rEOF')
3987 txt = self.TextIOWrapper(raw, 'ascii', newline='\n')
3988 txt.reconfigure(newline=None)
3989 self.assertEqual(txt.readline(), 'CR\n')
3990 raw = self.BytesIO(b'CR\rEOF')
3991 txt = self.TextIOWrapper(raw, 'ascii', newline='\n')
3992 txt.reconfigure(newline='')
3993 self.assertEqual(txt.readline(), 'CR\r')
3994 raw = self.BytesIO(b'CR\rLF\nEOF')
3995 txt = self.TextIOWrapper(raw, 'ascii', newline='\r')
3996 txt.reconfigure(newline='\n')
3997 self.assertEqual(txt.readline(), 'CR\rLF\n')
3998 raw = self.BytesIO(b'LF\nCR\rEOF')
3999 txt = self.TextIOWrapper(raw, 'ascii', newline='\n')
4000 txt.reconfigure(newline='\r')
4001 self.assertEqual(txt.readline(), 'LF\nCR\r')
4002 raw = self.BytesIO(b'CR\rCRLF\r\nEOF')
4003 txt = self.TextIOWrapper(raw, 'ascii', newline='\r')
4004 txt.reconfigure(newline='\r\n')
4005 self.assertEqual(txt.readline(), 'CR\rCRLF\r\n')
4006
4007 txt = self.TextIOWrapper(self.BytesIO(), 'ascii', newline='\r')
4008 txt.reconfigure(newline=None)
4009 txt.write('linesep\n')
4010 txt.reconfigure(newline='')
4011 txt.write('LF\n')
4012 txt.reconfigure(newline='\n')
4013 txt.write('LF\n')
4014 txt.reconfigure(newline='\r')
4015 txt.write('CR\n')
4016 txt.reconfigure(newline='\r\n')
4017 txt.write('CRLF\n')
4018 expected = 'linesep' + os.linesep + 'LF\nLF\nCR\rCRLF\r\n'
4019 self.assertEqual(txt.detach().getvalue().decode('ascii'), expected)
4020
4021 def test_issue25862(self):
4022 # Assertion failures occurred in tell() after read() and write().
4023 t = self.TextIOWrapper(self.BytesIO(b'test'), encoding='ascii')
4024 t.read(1)
4025 t.read()
4026 t.tell()
4027 t = self.TextIOWrapper(self.BytesIO(b'test'), encoding='ascii')
4028 t.read(1)
4029 t.write('x')
4030 t.tell()
4031
4032 def test_issue35928(self):
4033 p = self.BufferedRWPair(self.BytesIO(b'foo\nbar\n'), self.BytesIO())
4034 f = self.TextIOWrapper(p)
4035 res = f.readline()
4036 self.assertEqual(res, 'foo\n')
4037 f.write(res)
4038 self.assertEqual(res + f.readline(), 'foo\nbar\n')
4039
4040 def test_pickling_subclass(self):
4041 global MyTextIO
4042 class MyTextIO(self.TextIOWrapper):
4043 def __init__(self, raw, tag):
4044 super().__init__(raw)
4045 self.tag = tag
4046 def __getstate__(self):
4047 return self.tag, self.buffer.getvalue()
4048 def __setstate__(slf, state):
4049 tag, value = state
4050 slf.__init__(self.BytesIO(value), tag)
4051
4052 raw = self.BytesIO(b'data')
4053 txt = MyTextIO(raw, 'ham')
4054 for proto in range(pickle.HIGHEST_PROTOCOL + 1):
4055 with self.subTest(protocol=proto):
4056 pickled = pickle.dumps(txt, proto)
4057 newtxt = pickle.loads(pickled)
4058 self.assertEqual(newtxt.buffer.getvalue(), b'data')
4059 self.assertEqual(newtxt.tag, 'ham')
4060 del MyTextIO
4061
4062 @unittest.skipUnless(hasattr(os, "pipe"), "requires os.pipe()")
4063 def test_read_non_blocking(self):
4064 import os
4065 r, w = os.pipe()
4066 try:
4067 os.set_blocking(r, False)
4068 with self.io.open(r, 'rt') as textfile:
4069 r = None
4070 # Nothing has been written so a non-blocking read raises a BlockingIOError exception.
4071 with self.assertRaises(BlockingIOError):
4072 textfile.read()
4073 finally:
4074 if r is not None:
4075 os.close(r)
4076 os.close(w)
4077
4078
4079 class MemviewBytesIO(io.BytesIO):
4080 '''A BytesIO object whose read method returns memoryviews
4081 rather than bytes'''
4082
4083 def read1(self, len_):
4084 return _to_memoryview(super().read1(len_))
4085
4086 def read(self, len_):
4087 return _to_memoryview(super().read(len_))
4088
4089 def _to_memoryview(buf):
4090 '''Convert bytes-object *buf* to a non-trivial memoryview'''
4091
4092 arr = array.array('i')
4093 idx = len(buf) - len(buf) % arr.itemsize
4094 arr.frombytes(buf[:idx])
4095 return memoryview(arr)
4096
4097
4098 class CTextIOWrapperTest(TextIOWrapperTest):
4099 io = io
4100 shutdown_error = "LookupError: unknown encoding: ascii"
4101
4102 def test_initialization(self):
4103 r = self.BytesIO(b"\xc3\xa9\n\n")
4104 b = self.BufferedReader(r, 1000)
4105 t = self.TextIOWrapper(b, encoding="utf-8")
4106 self.assertRaises(ValueError, t.__init__, b, encoding="utf-8", newline='xyzzy')
4107 self.assertRaises(ValueError, t.read)
4108
4109 t = self.TextIOWrapper.__new__(self.TextIOWrapper)
4110 self.assertRaises(Exception, repr, t)
4111
4112 def test_garbage_collection(self):
4113 # C TextIOWrapper objects are collected, and collecting them flushes
4114 # all data to disk.
4115 # The Python version has __del__, so it ends in gc.garbage instead.
4116 with warnings_helper.check_warnings(('', ResourceWarning)):
4117 rawio = self.FileIO(os_helper.TESTFN, "wb")
4118 b = self.BufferedWriter(rawio)
4119 t = self.TextIOWrapper(b, encoding="ascii")
4120 t.write("456def")
4121 t.x = t
4122 wr = weakref.ref(t)
4123 del t
4124 support.gc_collect()
4125 self.assertIsNone(wr(), wr)
4126 with self.open(os_helper.TESTFN, "rb") as f:
4127 self.assertEqual(f.read(), b"456def")
4128
4129 def test_rwpair_cleared_before_textio(self):
4130 # Issue 13070: TextIOWrapper's finalization would crash when called
4131 # after the reference to the underlying BufferedRWPair's writer got
4132 # cleared by the GC.
4133 for i in range(1000):
4134 b1 = self.BufferedRWPair(self.MockRawIO(), self.MockRawIO())
4135 t1 = self.TextIOWrapper(b1, encoding="ascii")
4136 b2 = self.BufferedRWPair(self.MockRawIO(), self.MockRawIO())
4137 t2 = self.TextIOWrapper(b2, encoding="ascii")
4138 # circular references
4139 t1.buddy = t2
4140 t2.buddy = t1
4141 support.gc_collect()
4142
4143 def test_del__CHUNK_SIZE_SystemError(self):
4144 t = self.TextIOWrapper(self.BytesIO(), encoding='ascii')
4145 with self.assertRaises(AttributeError):
4146 del t._CHUNK_SIZE
4147
4148 def test_internal_buffer_size(self):
4149 # bpo-43260: TextIOWrapper's internal buffer should not store
4150 # data larger than chunk size.
4151 chunk_size = 8192 # default chunk size, updated later
4152
4153 class MockIO(self.MockRawIO):
4154 def write(self, data):
4155 if len(data) > chunk_size:
4156 raise RuntimeError
4157 return super().write(data)
4158
4159 buf = MockIO()
4160 t = self.TextIOWrapper(buf, encoding="ascii")
4161 chunk_size = t._CHUNK_SIZE
4162 t.write("abc")
4163 t.write("def")
4164 # default chunk size is 8192 bytes so t don't write data to buf.
4165 self.assertEqual([], buf._write_stack)
4166
4167 with self.assertRaises(RuntimeError):
4168 t.write("x"*(chunk_size+1))
4169
4170 self.assertEqual([b"abcdef"], buf._write_stack)
4171 t.write("ghi")
4172 t.write("x"*chunk_size)
4173 self.assertEqual([b"abcdef", b"ghi", b"x"*chunk_size], buf._write_stack)
4174
4175 def test_issue119506(self):
4176 chunk_size = 8192
4177
4178 class MockIO(self.MockRawIO):
4179 written = False
4180 def write(self, data):
4181 if not self.written:
4182 self.written = True
4183 t.write("middle")
4184 return super().write(data)
4185
4186 buf = MockIO()
4187 t = self.TextIOWrapper(buf)
4188 t.write("abc")
4189 t.write("def")
4190 # writing data which size >= chunk_size cause flushing buffer before write.
4191 t.write("g" * chunk_size)
4192 t.flush()
4193
4194 self.assertEqual([b"abcdef", b"middle", b"g"*chunk_size],
4195 buf._write_stack)
4196
4197 def test_issue142594(self):
4198 wrapper = None
4199 detached = False
4200 class ReentrantRawIO(self.RawIOBase):
4201 @property
4202 def closed(self):
4203 nonlocal detached
4204 if wrapper is not None and not detached:
4205 detached = True
4206 wrapper.detach()
4207 return False
4208
4209 raw = ReentrantRawIO()
4210 wrapper = self.TextIOWrapper(raw)
4211 wrapper.close() # should not crash
4212
4213
4214 class PyTextIOWrapperTest(TextIOWrapperTest):
4215 io = pyio
4216 shutdown_error = "LookupError: unknown encoding: ascii"
4217
4218
4219 class IncrementalNewlineDecoderTest(unittest.TestCase):
4220
4221 def check_newline_decoding_utf8(self, decoder):
4222 # UTF-8 specific tests for a newline decoder
4223 def _check_decode(b, s, **kwargs):
4224 # We exercise getstate() / setstate() as well as decode()
4225 state = decoder.getstate()
4226 self.assertEqual(decoder.decode(b, **kwargs), s)
4227 decoder.setstate(state)
4228 self.assertEqual(decoder.decode(b, **kwargs), s)
4229
4230 _check_decode(b'\xe8\xa2\x88', "\u8888")
4231
4232 _check_decode(b'\xe8', "")
4233 _check_decode(b'\xa2', "")
4234 _check_decode(b'\x88', "\u8888")
4235
4236 _check_decode(b'\xe8', "")
4237 _check_decode(b'\xa2', "")
4238 _check_decode(b'\x88', "\u8888")
4239
4240 _check_decode(b'\xe8', "")
4241 self.assertRaises(UnicodeDecodeError, decoder.decode, b'', final=True)
4242
4243 decoder.reset()
4244 _check_decode(b'\n', "\n")
4245 _check_decode(b'\r', "")
4246 _check_decode(b'', "\n", final=True)
4247 _check_decode(b'\r', "\n", final=True)
4248
4249 _check_decode(b'\r', "")
4250 _check_decode(b'a', "\na")
4251
4252 _check_decode(b'\r\r\n', "\n\n")
4253 _check_decode(b'\r', "")
4254 _check_decode(b'\r', "\n")
4255 _check_decode(b'\na', "\na")
4256
4257 _check_decode(b'\xe8\xa2\x88\r\n', "\u8888\n")
4258 _check_decode(b'\xe8\xa2\x88', "\u8888")
4259 _check_decode(b'\n', "\n")
4260 _check_decode(b'\xe8\xa2\x88\r', "\u8888")
4261 _check_decode(b'\n', "\n")
4262
4263 def check_newline_decoding(self, decoder, encoding):
4264 result = []
4265 if encoding is not None:
4266 encoder = codecs.getincrementalencoder(encoding)()
4267 def _decode_bytewise(s):
4268 # Decode one byte at a time
4269 for b in encoder.encode(s):
4270 result.append(decoder.decode(bytes([b])))
4271 else:
4272 encoder = None
4273 def _decode_bytewise(s):
4274 # Decode one char at a time
4275 for c in s:
4276 result.append(decoder.decode(c))
4277 self.assertEqual(decoder.newlines, None)
4278 _decode_bytewise("abc\n\r")
4279 self.assertEqual(decoder.newlines, '\n')
4280 _decode_bytewise("\nabc")
4281 self.assertEqual(decoder.newlines, ('\n', '\r\n'))
4282 _decode_bytewise("abc\r")
4283 self.assertEqual(decoder.newlines, ('\n', '\r\n'))
4284 _decode_bytewise("abc")
4285 self.assertEqual(decoder.newlines, ('\r', '\n', '\r\n'))
4286 _decode_bytewise("abc\r")
4287 self.assertEqual("".join(result), "abc\n\nabcabc\nabcabc")
4288 decoder.reset()
4289 input = "abc"
4290 if encoder is not None:
4291 encoder.reset()
4292 input = encoder.encode(input)
4293 self.assertEqual(decoder.decode(input), "abc")
4294 self.assertEqual(decoder.newlines, None)
4295
4296 def test_newline_decoder(self):
4297 encodings = (
4298 # None meaning the IncrementalNewlineDecoder takes unicode input
4299 # rather than bytes input
4300 None, 'utf-8', 'latin-1',
4301 'utf-16', 'utf-16-le', 'utf-16-be',
4302 'utf-32', 'utf-32-le', 'utf-32-be',
4303 )
4304 for enc in encodings:
4305 decoder = enc and codecs.getincrementaldecoder(enc)()
4306 decoder = self.IncrementalNewlineDecoder(decoder, translate=True)
4307 self.check_newline_decoding(decoder, enc)
4308 decoder = codecs.getincrementaldecoder("utf-8")()
4309 decoder = self.IncrementalNewlineDecoder(decoder, translate=True)
4310 self.check_newline_decoding_utf8(decoder)
4311 self.assertRaises(TypeError, decoder.setstate, 42)
4312
4313 def test_newline_bytes(self):
4314 # Issue 5433: Excessive optimization in IncrementalNewlineDecoder
4315 def _check(dec):
4316 self.assertEqual(dec.newlines, None)
4317 self.assertEqual(dec.decode("\u0D00"), "\u0D00")
4318 self.assertEqual(dec.newlines, None)
4319 self.assertEqual(dec.decode("\u0A00"), "\u0A00")
4320 self.assertEqual(dec.newlines, None)
4321 dec = self.IncrementalNewlineDecoder(None, translate=False)
4322 _check(dec)
4323 dec = self.IncrementalNewlineDecoder(None, translate=True)
4324 _check(dec)
4325
4326 def test_translate(self):
4327 # issue 35062
4328 for translate in (-2, -1, 1, 2):
4329 decoder = codecs.getincrementaldecoder("utf-8")()
4330 decoder = self.IncrementalNewlineDecoder(decoder, translate)
4331 self.check_newline_decoding_utf8(decoder)
4332 decoder = codecs.getincrementaldecoder("utf-8")()
4333 decoder = self.IncrementalNewlineDecoder(decoder, translate=0)
4334 self.assertEqual(decoder.decode(b"\r\r\n"), "\r\r\n")
4335
4336 class CIncrementalNewlineDecoderTest(IncrementalNewlineDecoderTest):
4337 @support.cpython_only
4338 def test_uninitialized(self):
4339 uninitialized = self.IncrementalNewlineDecoder.__new__(
4340 self.IncrementalNewlineDecoder)
4341 self.assertRaises(ValueError, uninitialized.decode, b'bar')
4342 self.assertRaises(ValueError, uninitialized.getstate)
4343 self.assertRaises(ValueError, uninitialized.setstate, (b'foo', 0))
4344 self.assertRaises(ValueError, uninitialized.reset)
4345
4346
4347 class PyIncrementalNewlineDecoderTest(IncrementalNewlineDecoderTest):
4348 pass
4349
4350
4351 # XXX Tests for open()
4352
4353 class MiscIOTest(unittest.TestCase):
4354
4355 # for test__all__, actual values are set in subclasses
4356 name_of_module = None
4357 extra_exported = ()
4358 not_exported = ()
4359
4360 def tearDown(self):
4361 os_helper.unlink(os_helper.TESTFN)
4362
4363 def test___all__(self):
4364 support.check__all__(self, self.io, self.name_of_module,
4365 extra=self.extra_exported,
4366 not_exported=self.not_exported)
4367
4368 def test_attributes(self):
4369 f = self.open(os_helper.TESTFN, "wb", buffering=0)
4370 self.assertEqual(f.mode, "wb")
4371 f.close()
4372
4373 f = self.open(os_helper.TESTFN, "w+", encoding="utf-8")
4374 self.assertEqual(f.mode, "w+")
4375 self.assertEqual(f.buffer.mode, "rb+") # Does it really matter?
4376 self.assertEqual(f.buffer.raw.mode, "rb+")
4377
4378 g = self.open(f.fileno(), "wb", closefd=False)
4379 self.assertEqual(g.mode, "wb")
4380 self.assertEqual(g.raw.mode, "wb")
4381 self.assertEqual(g.name, f.fileno())
4382 self.assertEqual(g.raw.name, f.fileno())
4383 f.close()
4384 g.close()
4385
4386 def test_removed_u_mode(self):
4387 # bpo-37330: The "U" mode has been removed in Python 3.11
4388 for mode in ("U", "rU", "r+U"):
4389 with self.assertRaises(ValueError) as cm:
4390 self.open(os_helper.TESTFN, mode)
4391 self.assertIn('invalid mode', str(cm.exception))
4392
4393 @unittest.skipUnless(hasattr(os, "pipe"), "requires os.pipe()")
4394 def test_open_pipe_with_append(self):
4395 # bpo-27805: Ignore ESPIPE from lseek() in open().
4396 r, w = os.pipe()
4397 self.addCleanup(os.close, r)
4398 f = self.open(w, 'a', encoding="utf-8")
4399 self.addCleanup(f.close)
4400 # Check that the file is marked non-seekable. On Windows, however, lseek
4401 # somehow succeeds on pipes.
4402 if sys.platform != 'win32':
4403 self.assertFalse(f.seekable())
4404
4405 def test_io_after_close(self):
4406 for kwargs in [
4407 {"mode": "w"},
4408 {"mode": "wb"},
4409 {"mode": "w", "buffering": 1},
4410 {"mode": "w", "buffering": 2},
4411 {"mode": "wb", "buffering": 0},
4412 {"mode": "r"},
4413 {"mode": "rb"},
4414 {"mode": "r", "buffering": 1},
4415 {"mode": "r", "buffering": 2},
4416 {"mode": "rb", "buffering": 0},
4417 {"mode": "w+"},
4418 {"mode": "w+b"},
4419 {"mode": "w+", "buffering": 1},
4420 {"mode": "w+", "buffering": 2},
4421 {"mode": "w+b", "buffering": 0},
4422 ]:
4423 if "b" not in kwargs["mode"]:
4424 kwargs["encoding"] = "utf-8"
4425 f = self.open(os_helper.TESTFN, **kwargs)
4426 f.close()
4427 self.assertRaises(ValueError, f.flush)
4428 self.assertRaises(ValueError, f.fileno)
4429 self.assertRaises(ValueError, f.isatty)
4430 self.assertRaises(ValueError, f.__iter__)
4431 if hasattr(f, "peek"):
4432 self.assertRaises(ValueError, f.peek, 1)
4433 self.assertRaises(ValueError, f.read)
4434 if hasattr(f, "read1"):
4435 self.assertRaises(ValueError, f.read1, 1024)
4436 self.assertRaises(ValueError, f.read1)
4437 if hasattr(f, "readall"):
4438 self.assertRaises(ValueError, f.readall)
4439 if hasattr(f, "readinto"):
4440 self.assertRaises(ValueError, f.readinto, bytearray(1024))
4441 if hasattr(f, "readinto1"):
4442 self.assertRaises(ValueError, f.readinto1, bytearray(1024))
4443 self.assertRaises(ValueError, f.readline)
4444 self.assertRaises(ValueError, f.readlines)
4445 self.assertRaises(ValueError, f.readlines, 1)
4446 self.assertRaises(ValueError, f.seek, 0)
4447 self.assertRaises(ValueError, f.tell)
4448 self.assertRaises(ValueError, f.truncate)
4449 self.assertRaises(ValueError, f.write,
4450 b"" if "b" in kwargs['mode'] else "")
4451 self.assertRaises(ValueError, f.writelines, [])
4452 self.assertRaises(ValueError, next, f)
4453
4454 def test_blockingioerror(self):
4455 # Various BlockingIOError issues
4456 class C(str):
4457 pass
4458 c = C("")
4459 b = self.BlockingIOError(1, c)
4460 c.b = b
4461 b.c = c
4462 wr = weakref.ref(c)
4463 del c, b
4464 support.gc_collect()
4465 self.assertIsNone(wr(), wr)
4466
4467 def test_abcs(self):
4468 # Test the visible base classes are ABCs.
4469 self.assertIsInstance(self.IOBase, abc.ABCMeta)
4470 self.assertIsInstance(self.RawIOBase, abc.ABCMeta)
4471 self.assertIsInstance(self.BufferedIOBase, abc.ABCMeta)
4472 self.assertIsInstance(self.TextIOBase, abc.ABCMeta)
4473
4474 def _check_abc_inheritance(self, abcmodule):
4475 with self.open(os_helper.TESTFN, "wb", buffering=0) as f:
4476 self.assertIsInstance(f, abcmodule.IOBase)
4477 self.assertIsInstance(f, abcmodule.RawIOBase)
4478 self.assertNotIsInstance(f, abcmodule.BufferedIOBase)
4479 self.assertNotIsInstance(f, abcmodule.TextIOBase)
4480 with self.open(os_helper.TESTFN, "wb") as f:
4481 self.assertIsInstance(f, abcmodule.IOBase)
4482 self.assertNotIsInstance(f, abcmodule.RawIOBase)
4483 self.assertIsInstance(f, abcmodule.BufferedIOBase)
4484 self.assertNotIsInstance(f, abcmodule.TextIOBase)
4485 with self.open(os_helper.TESTFN, "w", encoding="utf-8") as f:
4486 self.assertIsInstance(f, abcmodule.IOBase)
4487 self.assertNotIsInstance(f, abcmodule.RawIOBase)
4488 self.assertNotIsInstance(f, abcmodule.BufferedIOBase)
4489 self.assertIsInstance(f, abcmodule.TextIOBase)
4490
4491 def test_abc_inheritance(self):
4492 # Test implementations inherit from their respective ABCs
4493 self._check_abc_inheritance(self)
4494
4495 def test_abc_inheritance_official(self):
4496 # Test implementations inherit from the official ABCs of the
4497 # baseline "io" module.
4498 self._check_abc_inheritance(io)
4499
4500 def _check_warn_on_dealloc(self, *args, **kwargs):
4501 f = self.open(*args, **kwargs)
4502 r = repr(f)
4503 with self.assertWarns(ResourceWarning) as cm:
4504 f = None
4505 support.gc_collect()
4506 self.assertIn(r, str(cm.warning.args[0]))
4507
4508 def test_warn_on_dealloc(self):
4509 self._check_warn_on_dealloc(os_helper.TESTFN, "wb", buffering=0)
4510 self._check_warn_on_dealloc(os_helper.TESTFN, "wb")
4511 self._check_warn_on_dealloc(os_helper.TESTFN, "w", encoding="utf-8")
4512
4513 def _check_warn_on_dealloc_fd(self, *args, **kwargs):
4514 fds = []
4515 def cleanup_fds():
4516 for fd in fds:
4517 try:
4518 os.close(fd)
4519 except OSError as e:
4520 if e.errno != errno.EBADF:
4521 raise
4522 self.addCleanup(cleanup_fds)
4523 r, w = os.pipe()
4524 fds += r, w
4525 self._check_warn_on_dealloc(r, *args, **kwargs)
4526 # When using closefd=False, there's no warning
4527 r, w = os.pipe()
4528 fds += r, w
4529 with warnings_helper.check_no_resource_warning(self):
4530 self.open(r, *args, closefd=False, **kwargs)
4531
4532 @unittest.skipUnless(hasattr(os, "pipe"), "requires os.pipe()")
4533 def test_warn_on_dealloc_fd(self):
4534 self._check_warn_on_dealloc_fd("rb", buffering=0)
4535 self._check_warn_on_dealloc_fd("rb")
4536 self._check_warn_on_dealloc_fd("r", encoding="utf-8")
4537
4538
4539 def test_pickling(self):
4540 # Pickling file objects is forbidden
4541 msg = "cannot pickle"
4542 for kwargs in [
4543 {"mode": "w"},
4544 {"mode": "wb"},
4545 {"mode": "wb", "buffering": 0},
4546 {"mode": "r"},
4547 {"mode": "rb"},
4548 {"mode": "rb", "buffering": 0},
4549 {"mode": "w+"},
4550 {"mode": "w+b"},
4551 {"mode": "w+b", "buffering": 0},
4552 ]:
4553 if "b" not in kwargs["mode"]:
4554 kwargs["encoding"] = "utf-8"
4555 for protocol in range(pickle.HIGHEST_PROTOCOL + 1):
4556 with self.subTest(protocol=protocol, kwargs=kwargs):
4557 with self.open(os_helper.TESTFN, **kwargs) as f:
4558 with self.assertRaisesRegex(TypeError, msg):
4559 pickle.dumps(f, protocol)
4560
4561 @unittest.skipIf(support.is_emscripten, "Emscripten corrupts memory when writing to nonblocking fd")
4562 def test_nonblock_pipe_write_bigbuf(self):
4563 self._test_nonblock_pipe_write(16*1024)
4564
4565 @unittest.skipIf(support.is_emscripten, "Emscripten corrupts memory when writing to nonblocking fd")
4566 def test_nonblock_pipe_write_smallbuf(self):
4567 self._test_nonblock_pipe_write(1024)
4568
4569 @unittest.skipUnless(hasattr(os, 'set_blocking'),
4570 'os.set_blocking() required for this test')
4571 @unittest.skipUnless(hasattr(os, "pipe"), "requires os.pipe()")
4572 def _test_nonblock_pipe_write(self, bufsize):
4573 sent = []
4574 received = []
4575 r, w = os.pipe()
4576 os.set_blocking(r, False)
4577 os.set_blocking(w, False)
4578
4579 # To exercise all code paths in the C implementation we need
4580 # to play with buffer sizes. For instance, if we choose a
4581 # buffer size less than or equal to _PIPE_BUF (4096 on Linux)
4582 # then we will never get a partial write of the buffer.
4583 rf = self.open(r, mode='rb', closefd=True, buffering=bufsize)
4584 wf = self.open(w, mode='wb', closefd=True, buffering=bufsize)
4585
4586 with rf, wf:
4587 for N in 9999, 73, 7574:
4588 try:
4589 i = 0
4590 while True:
4591 msg = bytes([i % 26 + 97]) * N
4592 sent.append(msg)
4593 wf.write(msg)
4594 i += 1
4595
4596 except self.BlockingIOError as e:
4597 self.assertEqual(e.args[0], errno.EAGAIN)
4598 self.assertEqual(e.args[2], e.characters_written)
4599 sent[-1] = sent[-1][:e.characters_written]
4600 received.append(rf.read())
4601 msg = b'BLOCKED'
4602 wf.write(msg)
4603 sent.append(msg)
4604
4605 while True:
4606 try:
4607 wf.flush()
4608 break
4609 except self.BlockingIOError as e:
4610 self.assertEqual(e.args[0], errno.EAGAIN)
4611 self.assertEqual(e.args[2], e.characters_written)
4612 self.assertEqual(e.characters_written, 0)
4613 received.append(rf.read())
4614
4615 received += iter(rf.read, None)
4616
4617 sent, received = b''.join(sent), b''.join(received)
4618 self.assertEqual(sent, received)
4619 self.assertTrue(wf.closed)
4620 self.assertTrue(rf.closed)
4621
4622 def test_create_fail(self):
4623 # 'x' mode fails if file is existing
4624 with self.open(os_helper.TESTFN, 'w', encoding="utf-8"):
4625 pass
4626 self.assertRaises(FileExistsError, self.open, os_helper.TESTFN, 'x', encoding="utf-8")
4627
4628 def test_create_writes(self):
4629 # 'x' mode opens for writing
4630 with self.open(os_helper.TESTFN, 'xb') as f:
4631 f.write(b"spam")
4632 with self.open(os_helper.TESTFN, 'rb') as f:
4633 self.assertEqual(b"spam", f.read())
4634
4635 def test_open_allargs(self):
4636 # there used to be a buffer overflow in the parser for rawmode
4637 self.assertRaises(ValueError, self.open, os_helper.TESTFN, 'rwax+', encoding="utf-8")
4638
4639 def test_check_encoding_errors(self):
4640 # bpo-37388: open() and TextIOWrapper must check encoding and errors
4641 # arguments in dev mode
4642 mod = self.io.__name__
4643 filename = __file__
4644 invalid = 'Boom, Shaka Laka, Boom!'
4645 code = textwrap.dedent(f'''
4646 import sys
4647 from {mod} import open, TextIOWrapper
4648
4649 try:
4650 open({filename!r}, encoding={invalid!r})
4651 except LookupError:
4652 pass
4653 else:
4654 sys.exit(21)
4655
4656 try:
4657 open({filename!r}, errors={invalid!r})
4658 except LookupError:
4659 pass
4660 else:
4661 sys.exit(22)
4662
4663 fp = open({filename!r}, "rb")
4664 with fp:
4665 try:
4666 TextIOWrapper(fp, encoding={invalid!r})
4667 except LookupError:
4668 pass
4669 else:
4670 sys.exit(23)
4671
4672 try:
4673 TextIOWrapper(fp, errors={invalid!r})
4674 except LookupError:
4675 pass
4676 else:
4677 sys.exit(24)
4678
4679 sys.exit(10)
4680 ''')
4681 proc = assert_python_failure('-X', 'dev', '-c', code)
4682 self.assertEqual(proc.rc, 10, proc)
4683
4684 def test_check_encoding_warning(self):
4685 # PEP 597: Raise warning when encoding is not specified
4686 # and sys.flags.warn_default_encoding is set.
4687 mod = self.io.__name__
4688 filename = __file__
4689 code = textwrap.dedent(f'''\
4690 import sys
4691 from {mod} import open, TextIOWrapper
4692 import pathlib
4693
4694 with open({filename!r}) as f: # line 5
4695 pass
4696
4697 pathlib.Path({filename!r}).read_text() # line 8
4698 ''')
4699 proc = assert_python_ok('-X', 'warn_default_encoding', '-c', code)
4700 warnings = proc.err.splitlines()
4701 self.assertEqual(len(warnings), 2)
4702 self.assertStartsWith(warnings[0], b"<string>:5: EncodingWarning: ")
4703 self.assertStartsWith(warnings[1], b"<string>:8: EncodingWarning: ")
4704
4705 def test_text_encoding(self):
4706 # PEP 597, bpo-47000. io.text_encoding() returns "locale" or "utf-8"
4707 # based on sys.flags.utf8_mode
4708 code = "import io; print(io.text_encoding(None))"
4709
4710 proc = assert_python_ok('-X', 'utf8=0', '-c', code)
4711 self.assertEqual(b"locale", proc.out.strip())
4712
4713 proc = assert_python_ok('-X', 'utf8=1', '-c', code)
4714 self.assertEqual(b"utf-8", proc.out.strip())
4715
4716
4717 class CMiscIOTest(MiscIOTest):
4718 io = io
4719 name_of_module = "io", "_io"
4720 extra_exported = "BlockingIOError",
4721
4722 def test_readinto_buffer_overflow(self):
4723 # Issue #18025
4724 class BadReader(self.io.BufferedIOBase):
4725 def read(self, n=-1):
4726 return b'x' * 10**6
4727 bufio = BadReader()
4728 b = bytearray(2)
4729 self.assertRaises(ValueError, bufio.readinto, b)
4730
4731 def check_daemon_threads_shutdown_deadlock(self, stream_name):
4732 # Issue #23309: deadlocks at shutdown should be avoided when a
4733 # daemon thread and the main thread both write to a file.
4734 code = """if 1:
4735 import sys
4736 import time
4737 import threading
4738 from test.support import SuppressCrashReport
4739
4740 file = sys.{stream_name}
4741
4742 def run():
4743 while True:
4744 file.write('.')
4745 file.flush()
4746
4747 crash = SuppressCrashReport()
4748 crash.__enter__()
4749 # don't call __exit__(): the crash occurs at Python shutdown
4750
4751 thread = threading.Thread(target=run)
4752 thread.daemon = True
4753 thread.start()
4754
4755 time.sleep(0.5)
4756 file.write('!')
4757 file.flush()
4758 """.format_map(locals())
4759 res, _ = run_python_until_end("-c", code)
4760 err = res.err.decode()
4761 if res.rc != 0:
4762 # Failure: should be a fatal error
4763 pattern = (r"Fatal Python error: _enter_buffered_busy: "
4764 r"could not acquire lock "
4765 r"for <(_io\.)?BufferedWriter name='<{stream_name}>'> "
4766 r"at interpreter shutdown, possibly due to "
4767 r"daemon threads".format_map(locals()))
4768 self.assertRegex(err, pattern)
4769 else:
4770 self.assertFalse(err.strip('.!'))
4771
4772 @threading_helper.requires_working_threading()
4773 @support.requires_resource('walltime')
4774 def test_daemon_threads_shutdown_stdout_deadlock(self):
4775 self.check_daemon_threads_shutdown_deadlock('stdout')
4776
4777 @threading_helper.requires_working_threading()
4778 @support.requires_resource('walltime')
4779 def test_daemon_threads_shutdown_stderr_deadlock(self):
4780 self.check_daemon_threads_shutdown_deadlock('stderr')
4781
4782
4783 class PyMiscIOTest(MiscIOTest):
4784 io = pyio
4785 name_of_module = "_pyio", "io"
4786 extra_exported = "BlockingIOError", "open_code",
4787 not_exported = "valid_seek_flags",
4788
4789
4790 @unittest.skipIf(os.name == 'nt', 'POSIX signals required for this test.')
4791 class SignalsTest(unittest.TestCase):
4792
4793 def setUp(self):
4794 self.oldalrm = signal.signal(signal.SIGALRM, self.alarm_interrupt)
4795
4796 def tearDown(self):
4797 signal.signal(signal.SIGALRM, self.oldalrm)
4798
4799 def alarm_interrupt(self, sig, frame):
4800 1/0
4801
4802 def check_interrupted_write(self, item, bytes, **fdopen_kwargs):
4803 """Check that a partial write, when it gets interrupted, properly
4804 invokes the signal handler, and bubbles up the exception raised
4805 in the latter."""
4806
4807 # XXX This test has three flaws that appear when objects are
4808 # XXX not reference counted.
4809
4810 # - if wio.write() happens to trigger a garbage collection,
4811 # the signal exception may be raised when some __del__
4812 # method is running; it will not reach the assertRaises()
4813 # call.
4814
4815 # - more subtle, if the wio object is not destroyed at once
4816 # and survives this function, the next opened file is likely
4817 # to have the same fileno (since the file descriptor was
4818 # actively closed). When wio.__del__ is finally called, it
4819 # will close the other's test file... To trigger this with
4820 # CPython, try adding "global wio" in this function.
4821
4822 # - This happens only for streams created by the _pyio module,
4823 # because a wio.close() that fails still consider that the
4824 # file needs to be closed again. You can try adding an
4825 # "assert wio.closed" at the end of the function.
4826
4827 # Fortunately, a little gc.collect() seems to be enough to
4828 # work around all these issues.
4829 support.gc_collect() # For PyPy or other GCs.
4830
4831 read_results = []
4832 def _read():
4833 s = os.read(r, 1)
4834 read_results.append(s)
4835
4836 t = threading.Thread(target=_read)
4837 t.daemon = True
4838 r, w = os.pipe()
4839 fdopen_kwargs["closefd"] = False
4840 large_data = item * (support.PIPE_MAX_SIZE // len(item) + 1)
4841 try:
4842 wio = self.io.open(w, **fdopen_kwargs)
4843 if hasattr(signal, 'pthread_sigmask'):
4844 # create the thread with SIGALRM signal blocked
4845 signal.pthread_sigmask(signal.SIG_BLOCK, [signal.SIGALRM])
4846 t.start()
4847 signal.pthread_sigmask(signal.SIG_UNBLOCK, [signal.SIGALRM])
4848 else:
4849 t.start()
4850
4851 # Fill the pipe enough that the write will be blocking.
4852 # It will be interrupted by the timer armed above. Since the
4853 # other thread has read one byte, the low-level write will
4854 # return with a successful (partial) result rather than an EINTR.
4855 # The buffered IO layer must check for pending signal
4856 # handlers, which in this case will invoke alarm_interrupt().
4857 signal.alarm(1)
4858 try:
4859 self.assertRaises(ZeroDivisionError, wio.write, large_data)
4860 finally:
4861 signal.alarm(0)
4862 t.join()
4863 # We got one byte, get another one and check that it isn't a
4864 # repeat of the first one.
4865 read_results.append(os.read(r, 1))
4866 self.assertEqual(read_results, [bytes[0:1], bytes[1:2]])
4867 finally:
4868 os.close(w)
4869 os.close(r)
4870 # This is deliberate. If we didn't close the file descriptor
4871 # before closing wio, wio would try to flush its internal
4872 # buffer, and block again.
4873 try:
4874 wio.close()
4875 except OSError as e:
4876 if e.errno != errno.EBADF:
4877 raise
4878
4879 @requires_alarm
4880 @unittest.skipUnless(hasattr(os, "pipe"), "requires os.pipe()")
4881 def test_interrupted_write_unbuffered(self):
4882 self.check_interrupted_write(b"xy", b"xy", mode="wb", buffering=0)
4883
4884 @requires_alarm
4885 @unittest.skipUnless(hasattr(os, "pipe"), "requires os.pipe()")
4886 def test_interrupted_write_buffered(self):
4887 self.check_interrupted_write(b"xy", b"xy", mode="wb")
4888
4889 @requires_alarm
4890 @unittest.skipUnless(hasattr(os, "pipe"), "requires os.pipe()")
4891 def test_interrupted_write_text(self):
4892 self.check_interrupted_write("xy", b"xy", mode="w", encoding="ascii")
4893
4894 @support.no_tracing
4895 def check_reentrant_write(self, data, **fdopen_kwargs):
4896 def on_alarm(*args):
4897 # Will be called reentrantly from the same thread
4898 wio.write(data)
4899 1/0
4900 signal.signal(signal.SIGALRM, on_alarm)
4901 r, w = os.pipe()
4902 wio = self.io.open(w, **fdopen_kwargs)
4903 try:
4904 signal.alarm(1)
4905 # Either the reentrant call to wio.write() fails with RuntimeError,
4906 # or the signal handler raises ZeroDivisionError.
4907 with self.assertRaises((ZeroDivisionError, RuntimeError)) as cm:
4908 while 1:
4909 for i in range(100):
4910 wio.write(data)
4911 wio.flush()
4912 # Make sure the buffer doesn't fill up and block further writes
4913 os.read(r, len(data) * 100)
4914 exc = cm.exception
4915 if isinstance(exc, RuntimeError):
4916 self.assertStartsWith(str(exc), "reentrant call")
4917 finally:
4918 signal.alarm(0)
4919 wio.close()
4920 os.close(r)
4921
4922 @requires_alarm
4923 def test_reentrant_write_buffered(self):
4924 self.check_reentrant_write(b"xy", mode="wb")
4925
4926 @requires_alarm
4927 def test_reentrant_write_text(self):
4928 self.check_reentrant_write("xy", mode="w", encoding="ascii")
4929
4930 def check_interrupted_read_retry(self, decode, **fdopen_kwargs):
4931 """Check that a buffered read, when it gets interrupted (either
4932 returning a partial result or EINTR), properly invokes the signal
4933 handler and retries if the latter returned successfully."""
4934 r, w = os.pipe()
4935 fdopen_kwargs["closefd"] = False
4936 def alarm_handler(sig, frame):
4937 os.write(w, b"bar")
4938 signal.signal(signal.SIGALRM, alarm_handler)
4939 try:
4940 rio = self.io.open(r, **fdopen_kwargs)
4941 os.write(w, b"foo")
4942 signal.alarm(1)
4943 # Expected behaviour:
4944 # - first raw read() returns partial b"foo"
4945 # - second raw read() returns EINTR
4946 # - third raw read() returns b"bar"
4947 self.assertEqual(decode(rio.read(6)), "foobar")
4948 finally:
4949 signal.alarm(0)
4950 rio.close()
4951 os.close(w)
4952 os.close(r)
4953
4954 @requires_alarm
4955 @support.requires_resource('walltime')
4956 def test_interrupted_read_retry_buffered(self):
4957 self.check_interrupted_read_retry(lambda x: x.decode('latin1'),
4958 mode="rb")
4959
4960 @requires_alarm
4961 @support.requires_resource('walltime')
4962 def test_interrupted_read_retry_text(self):
4963 self.check_interrupted_read_retry(lambda x: x,
4964 mode="r", encoding="latin1")
4965
4966 def check_interrupted_write_retry(self, item, **fdopen_kwargs):
4967 """Check that a buffered write, when it gets interrupted (either
4968 returning a partial result or EINTR), properly invokes the signal
4969 handler and retries if the latter returned successfully."""
4970 select = import_helper.import_module("select")
4971
4972 # A quantity that exceeds the buffer size of an anonymous pipe's
4973 # write end.
4974 N = support.PIPE_MAX_SIZE
4975 r, w = os.pipe()
4976 fdopen_kwargs["closefd"] = False
4977
4978 # We need a separate thread to read from the pipe and allow the
4979 # write() to finish. This thread is started after the SIGALRM is
4980 # received (forcing a first EINTR in write()).
4981 read_results = []
4982 write_finished = False
4983 error = None
4984 def _read():
4985 try:
4986 while not write_finished:
4987 while r in select.select([r], [], [], 1.0)[0]:
4988 s = os.read(r, 1024)
4989 read_results.append(s)
4990 except BaseException as exc:
4991 nonlocal error
4992 error = exc
4993 t = threading.Thread(target=_read)
4994 t.daemon = True
4995 def alarm1(sig, frame):
4996 signal.signal(signal.SIGALRM, alarm2)
4997 signal.alarm(1)
4998 def alarm2(sig, frame):
4999 t.start()
5000
5001 large_data = item * N
5002 signal.signal(signal.SIGALRM, alarm1)
5003 try:
5004 wio = self.io.open(w, **fdopen_kwargs)
5005 signal.alarm(1)
5006 # Expected behaviour:
5007 # - first raw write() is partial (because of the limited pipe buffer
5008 # and the first alarm)
5009 # - second raw write() returns EINTR (because of the second alarm)
5010 # - subsequent write()s are successful (either partial or complete)
5011 written = wio.write(large_data)
5012 self.assertEqual(N, written)
5013
5014 wio.flush()
5015 write_finished = True
5016 t.join()
5017
5018 self.assertIsNone(error)
5019 self.assertEqual(N, sum(len(x) for x in read_results))
5020 finally:
5021 signal.alarm(0)
5022 write_finished = True
5023 os.close(w)
5024 os.close(r)
5025 # This is deliberate. If we didn't close the file descriptor
5026 # before closing wio, wio would try to flush its internal
5027 # buffer, and could block (in case of failure).
5028 try:
5029 wio.close()
5030 except OSError as e:
5031 if e.errno != errno.EBADF:
5032 raise
5033
5034 @requires_alarm
5035 @support.requires_resource('walltime')
5036 def test_interrupted_write_retry_buffered(self):
5037 self.check_interrupted_write_retry(b"x", mode="wb")
5038
5039 @requires_alarm
5040 @support.requires_resource('walltime')
5041 def test_interrupted_write_retry_text(self):
5042 self.check_interrupted_write_retry("x", mode="w", encoding="latin1")
5043
5044
5045 class CSignalsTest(SignalsTest):
5046 io = io
5047
5048 class PySignalsTest(SignalsTest):
5049 io = pyio
5050
5051 # Handling reentrancy issues would slow down _pyio even more, so the
5052 # tests are disabled.
5053 test_reentrant_write_buffered = None
5054 test_reentrant_write_text = None
5055
5056
5057 class ProtocolsTest(unittest.TestCase):
5058 class MyReader:
5059 def read(self, sz=-1):
5060 return b""
5061
5062 class MyWriter:
5063 def write(self, b: bytes):
5064 pass
5065
5066 def test_reader_subclass(self):
5067 self.assertIsSubclass(self.MyReader, io.Reader)
5068 self.assertNotIsSubclass(str, io.Reader)
5069
5070 def test_writer_subclass(self):
5071 self.assertIsSubclass(self.MyWriter, io.Writer)
5072 self.assertNotIsSubclass(str, io.Writer)
5073
5074
5075 def load_tests(loader, tests, pattern):
5076 tests = (CIOTest, PyIOTest, APIMismatchTest,
5077 CBufferedReaderTest, PyBufferedReaderTest,
5078 CBufferedWriterTest, PyBufferedWriterTest,
5079 CBufferedRWPairTest, PyBufferedRWPairTest,
5080 CBufferedRandomTest, PyBufferedRandomTest,
5081 StatefulIncrementalDecoderTest,
5082 CIncrementalNewlineDecoderTest, PyIncrementalNewlineDecoderTest,
5083 CTextIOWrapperTest, PyTextIOWrapperTest,
5084 CMiscIOTest, PyMiscIOTest,
5085 CSignalsTest, PySignalsTest, TestIOCTypes,
5086 ProtocolsTest,
5087 )
5088
5089 # Put the namespaces of the IO module we are testing and some useful mock
5090 # classes in the __dict__ of each test.
5091 mocks = (MockRawIO, MisbehavedRawIO, MockFileIO, CloseFailureIO,
5092 MockNonBlockWriterIO, MockUnseekableIO, MockRawIOWithoutRead,
5093 SlowFlushRawIO, MockCharPseudoDevFileIO)
5094 all_members = io.__all__
5095 c_io_ns = {name : getattr(io, name) for name in all_members}
5096 py_io_ns = {name : getattr(pyio, name) for name in all_members}
5097 globs = globals()
5098 c_io_ns.update((x.__name__, globs["C" + x.__name__]) for x in mocks)
5099 py_io_ns.update((x.__name__, globs["Py" + x.__name__]) for x in mocks)
5100 for test in tests:
5101 if test.__name__.startswith("C"):
5102 for name, obj in c_io_ns.items():
5103 setattr(test, name, obj)
5104 test.is_C = True
5105 elif test.__name__.startswith("Py"):
5106 for name, obj in py_io_ns.items():
5107 setattr(test, name, obj)
5108 test.is_C = False
5109
5110 suite = loader.suiteClass()
5111 for test in tests:
5112 suite.addTest(loader.loadTestsFromTestCase(test))
5113 return suite
5114
5115 if __name__ == "__main__":
5116 unittest.main()