Debian Codegraph @

Lib/test/test_largefile.py

raw
1 """Test largefile support on system where this makes sense.
2 """
3
4 import os
5 import sys
6 import unittest
7 import socket
8 import shutil
9 import threading
10 from test.support import requires, bigmemtest, requires_resource
11 from test.support import SHORT_TIMEOUT
12 from test.support import socket_helper
13 from test.support.os_helper import TESTFN, unlink
14 import io # C implementation of io
15 import _pyio as pyio # Python implementation of io
16
17 # size of file to create (>2 GiB; 2 GiB == 2,147,483,648 bytes)
18 size = 2_500_000_000
19 TESTFN2 = TESTFN + '2'
20
21
22 class LargeFileTest:
23
24 def setUp(self):
25 if os.path.exists(TESTFN):
26 mode = 'r+b'
27 else:
28 mode = 'w+b'
29
30 with self.open(TESTFN, mode) as f:
31 current_size = os.fstat(f.fileno()).st_size
32 if current_size == size+1:
33 return
34
35 if current_size == 0:
36 f.write(b'z')
37
38 f.seek(0)
39 f.seek(size)
40 f.write(b'a')
41 f.flush()
42 self.assertEqual(os.fstat(f.fileno()).st_size, size+1)
43
44 @classmethod
45 def tearDownClass(cls):
46 with cls.open(TESTFN, 'wb'):
47 pass
48 if not os.stat(TESTFN).st_size == 0:
49 raise cls.failureException('File was not truncated by opening '
50 'with mode "wb"')
51 unlink(TESTFN2)
52
53
54 class TestFileMethods(LargeFileTest):
55 """Test that each file function works as expected for large
56 (i.e. > 2 GiB) files.
57 """
58
59 # _pyio.FileIO.readall() uses a temporary bytearray then casted to bytes,
60 # so memuse=2 is needed
61 @bigmemtest(size=size, memuse=2, dry_run=False)
62 def test_large_read(self, _size):
63 # bpo-24658: Test that a read greater than 2GB does not fail.
64 with self.open(TESTFN, "rb") as f:
65 self.assertEqual(len(f.read()), size + 1)
66 self.assertEqual(f.tell(), size + 1)
67
68 def test_osstat(self):
69 self.assertEqual(os.stat(TESTFN).st_size, size+1)
70
71 def test_seek_read(self):
72 with self.open(TESTFN, 'rb') as f:
73 self.assertEqual(f.tell(), 0)
74 self.assertEqual(f.read(1), b'z')
75 self.assertEqual(f.tell(), 1)
76 f.seek(0)
77 self.assertEqual(f.tell(), 0)
78 f.seek(0, 0)
79 self.assertEqual(f.tell(), 0)
80 f.seek(42)
81 self.assertEqual(f.tell(), 42)
82 f.seek(42, 0)
83 self.assertEqual(f.tell(), 42)
84 f.seek(42, 1)
85 self.assertEqual(f.tell(), 84)
86 f.seek(0, 1)
87 self.assertEqual(f.tell(), 84)
88 f.seek(0, 2) # seek from the end
89 self.assertEqual(f.tell(), size + 1 + 0)
90 f.seek(-10, 2)
91 self.assertEqual(f.tell(), size + 1 - 10)
92 f.seek(-size-1, 2)
93 self.assertEqual(f.tell(), 0)
94 f.seek(size)
95 self.assertEqual(f.tell(), size)
96 # the 'a' that was written at the end of file above
97 self.assertEqual(f.read(1), b'a')
98 f.seek(-size-1, 1)
99 self.assertEqual(f.read(1), b'z')
100 self.assertEqual(f.tell(), 1)
101
102 def test_lseek(self):
103 with self.open(TESTFN, 'rb') as f:
104 self.assertEqual(os.lseek(f.fileno(), 0, 0), 0)
105 self.assertEqual(os.lseek(f.fileno(), 42, 0), 42)
106 self.assertEqual(os.lseek(f.fileno(), 42, 1), 84)
107 self.assertEqual(os.lseek(f.fileno(), 0, 1), 84)
108 self.assertEqual(os.lseek(f.fileno(), 0, 2), size+1+0)
109 self.assertEqual(os.lseek(f.fileno(), -10, 2), size+1-10)
110 self.assertEqual(os.lseek(f.fileno(), -size-1, 2), 0)
111 self.assertEqual(os.lseek(f.fileno(), size, 0), size)
112 # the 'a' that was written at the end of file above
113 self.assertEqual(f.read(1), b'a')
114
115 def test_truncate(self):
116 with self.open(TESTFN, 'r+b') as f:
117 if not hasattr(f, 'truncate'):
118 raise unittest.SkipTest("open().truncate() not available "
119 "on this system")
120 f.seek(0, 2)
121 # else we've lost track of the true size
122 self.assertEqual(f.tell(), size+1)
123 # Cut it back via seek + truncate with no argument.
124 newsize = size - 10
125 f.seek(newsize)
126 f.truncate()
127 self.assertEqual(f.tell(), newsize) # else pointer moved
128 f.seek(0, 2)
129 self.assertEqual(f.tell(), newsize) # else wasn't truncated
130 # Ensure that truncate(smaller than true size) shrinks
131 # the file.
132 newsize -= 1
133 f.seek(42)
134 f.truncate(newsize)
135 self.assertEqual(f.tell(), 42)
136 f.seek(0, 2)
137 self.assertEqual(f.tell(), newsize)
138 # XXX truncate(larger than true size) is ill-defined
139 # across platform; cut it waaaaay back
140 f.seek(0)
141 f.truncate(1)
142 self.assertEqual(f.tell(), 0) # else pointer moved
143 f.seek(0)
144 # Verify readall on a truncated file is well behaved. read()
145 # without a size can be unbounded, this should get just the byte
146 # that remains.
147 self.assertEqual(len(f.read()), 1) # else wasn't truncated
148
149 def test_seekable(self):
150 # Issue #5016; seekable() can return False when the current position
151 # is negative when truncated to an int.
152 for pos in (2**31-1, 2**31, 2**31+1):
153 with self.open(TESTFN, 'rb') as f:
154 f.seek(pos)
155 self.assertTrue(f.seekable())
156
157 @bigmemtest(size=size, memuse=2, dry_run=False)
158 def test_seek_readall(self, _size):
159 # Seek which doesn't change position should readall successfully.
160 with self.open(TESTFN, 'rb') as f:
161 self.assertEqual(f.seek(0, os.SEEK_CUR), 0)
162 self.assertEqual(len(f.read()), size + 1)
163
164 # Seek which changes (or might change) position should readall
165 # successfully.
166 with self.open(TESTFN, 'rb') as f:
167 self.assertEqual(f.seek(20, os.SEEK_SET), 20)
168 self.assertEqual(len(f.read()), size - 19)
169
170 with self.open(TESTFN, 'rb') as f:
171 self.assertEqual(f.seek(-3, os.SEEK_END), size - 2)
172 self.assertEqual(len(f.read()), 3)
173
174 def skip_no_disk_space(path, required):
175 def decorator(fun):
176 def wrapper(*args, **kwargs):
177 if not hasattr(shutil, "disk_usage"):
178 raise unittest.SkipTest("requires shutil.disk_usage")
179 if shutil.disk_usage(os.path.realpath(path)).free < required:
180 hsize = int(required / 1024 / 1024)
181 raise unittest.SkipTest(
182 f"required {hsize} MiB of free disk space")
183 return fun(*args, **kwargs)
184 return wrapper
185 return decorator
186
187
188 class TestCopyfile(LargeFileTest, unittest.TestCase):
189 open = staticmethod(io.open)
190
191 # Exact required disk space would be (size * 2), but let's give it a
192 # bit more tolerance.
193 @skip_no_disk_space(TESTFN, size * 2.5)
194 @requires_resource('cpu')
195 def test_it(self):
196 # Internally shutil.copyfile() can use "fast copy" methods like
197 # os.sendfile().
198 size = os.path.getsize(TESTFN)
199 shutil.copyfile(TESTFN, TESTFN2)
200 self.assertEqual(os.path.getsize(TESTFN2), size)
201 with open(TESTFN2, 'rb') as f:
202 self.assertEqual(f.read(5), b'z\x00\x00\x00\x00')
203 f.seek(size - 5)
204 self.assertEqual(f.read(), b'\x00\x00\x00\x00a')
205
206
207 @unittest.skipIf(not hasattr(os, 'sendfile'), 'sendfile not supported')
208 class TestSocketSendfile(LargeFileTest, unittest.TestCase):
209 open = staticmethod(io.open)
210 timeout = SHORT_TIMEOUT
211
212 def setUp(self):
213 super().setUp()
214 self.thread = None
215
216 def tearDown(self):
217 super().tearDown()
218 if self.thread is not None:
219 self.thread.join(self.timeout)
220 self.thread = None
221
222 def tcp_server(self, sock):
223 def run(sock):
224 with sock:
225 conn, _ = sock.accept()
226 conn.settimeout(self.timeout)
227 with conn, open(TESTFN2, 'wb') as f:
228 event.wait(self.timeout)
229 while True:
230 chunk = conn.recv(65536)
231 if not chunk:
232 return
233 f.write(chunk)
234
235 event = threading.Event()
236 sock.settimeout(self.timeout)
237 self.thread = threading.Thread(target=run, args=(sock, ))
238 self.thread.start()
239 event.set()
240
241 # Exact required disk space would be (size * 2), but let's give it a
242 # bit more tolerance.
243 @skip_no_disk_space(TESTFN, size * 2.5)
244 @requires_resource('cpu')
245 def test_it(self):
246 port = socket_helper.find_unused_port()
247 with socket.create_server(("", port)) as sock:
248 self.tcp_server(sock)
249 with socket.create_connection(("127.0.0.1", port)) as client:
250 with open(TESTFN, 'rb') as f:
251 client.sendfile(f)
252 self.tearDown()
253
254 size = os.path.getsize(TESTFN)
255 self.assertEqual(os.path.getsize(TESTFN2), size)
256 with open(TESTFN2, 'rb') as f:
257 self.assertEqual(f.read(5), b'z\x00\x00\x00\x00')
258 f.seek(size - 5)
259 self.assertEqual(f.read(), b'\x00\x00\x00\x00a')
260
261
262 def setUpModule():
263 try:
264 import signal
265 # The default handler for SIGXFSZ is to abort the process.
266 # By ignoring it, system calls exceeding the file size resource
267 # limit will raise OSError instead of crashing the interpreter.
268 signal.signal(signal.SIGXFSZ, signal.SIG_IGN)
269 except (ImportError, AttributeError):
270 pass
271
272 # On Windows and Mac OSX this test consumes large resources; It
273 # takes a long time to build the >2 GiB file and takes >2 GiB of disk
274 # space therefore the resource must be enabled to run this test.
275 # If not, nothing after this line stanza will be executed.
276 if sys.platform[:3] == 'win' or sys.platform == 'darwin':
277 requires('largefile',
278 'test requires %s bytes and a long time to run' % str(size))
279 else:
280 # Only run if the current filesystem supports large files.
281 # (Skip this test on Windows, since we now always support
282 # large files.)
283 f = open(TESTFN, 'wb', buffering=0)
284 try:
285 # 2**31 == 2147483648
286 f.seek(2147483649)
287 # Seeking is not enough of a test: you must write and flush, too!
288 f.write(b'x')
289 f.flush()
290 except (OSError, OverflowError):
291 raise unittest.SkipTest("filesystem does not have "
292 "largefile support")
293 finally:
294 f.close()
295 unlink(TESTFN)
296
297
298 class CLargeFileTest(TestFileMethods, unittest.TestCase):
299 open = staticmethod(io.open)
300
301
302 class PyLargeFileTest(TestFileMethods, unittest.TestCase):
303 open = staticmethod(pyio.open)
304
305
306 def tearDownModule():
307 unlink(TESTFN)
308 unlink(TESTFN2)
309
310
311 if __name__ == '__main__':
312 unittest.main()