| 1 |
"""Test largefile support on system where this makes sense. |
| 2 |
""" |
| 3 |
|
| 4 |
import os |
| 5 |
import sys |
| 6 |
import unittest |
| 7 |
import socket |
| 8 |
import shutil |
| 9 |
import threading |
| 10 |
from test.support import requires, bigmemtest, requires_resource |
| 11 |
from test.support import SHORT_TIMEOUT |
| 12 |
from test.support import socket_helper |
| 13 |
from test.support.os_helper import TESTFN, unlink |
| 14 |
import io # C implementation of io |
| 15 |
import _pyio as pyio # Python implementation of io |
| 16 |
|
| 17 |
# size of file to create (>2 GiB; 2 GiB == 2,147,483,648 bytes) |
| 18 |
size = 2_500_000_000 |
| 19 |
TESTFN2 = TESTFN + '2' |
| 20 |
|
| 21 |
|
| 22 |
class LargeFileTest: |
| 23 |
|
| 24 |
def setUp(self): |
| 25 |
if os.path.exists(TESTFN): |
| 26 |
mode = 'r+b' |
| 27 |
else: |
| 28 |
mode = 'w+b' |
| 29 |
|
| 30 |
with self.open(TESTFN, mode) as f: |
| 31 |
current_size = os.fstat(f.fileno()).st_size |
| 32 |
if current_size == size+1: |
| 33 |
return |
| 34 |
|
| 35 |
if current_size == 0: |
| 36 |
f.write(b'z') |
| 37 |
|
| 38 |
f.seek(0) |
| 39 |
f.seek(size) |
| 40 |
f.write(b'a') |
| 41 |
f.flush() |
| 42 |
self.assertEqual(os.fstat(f.fileno()).st_size, size+1) |
| 43 |
|
| 44 |
@classmethod |
| 45 |
def tearDownClass(cls): |
| 46 |
with cls.open(TESTFN, 'wb'): |
| 47 |
pass |
| 48 |
if not os.stat(TESTFN).st_size == 0: |
| 49 |
raise cls.failureException('File was not truncated by opening ' |
| 50 |
'with mode "wb"') |
| 51 |
unlink(TESTFN2) |
| 52 |
|
| 53 |
|
| 54 |
class TestFileMethods(LargeFileTest): |
| 55 |
"""Test that each file function works as expected for large |
| 56 |
(i.e. > 2 GiB) files. |
| 57 |
""" |
| 58 |
|
| 59 |
# _pyio.FileIO.readall() uses a temporary bytearray then casted to bytes, |
| 60 |
# so memuse=2 is needed |
| 61 |
@bigmemtest(size=size, memuse=2, dry_run=False) |
| 62 |
def test_large_read(self, _size): |
| 63 |
# bpo-24658: Test that a read greater than 2GB does not fail. |
| 64 |
with self.open(TESTFN, "rb") as f: |
| 65 |
self.assertEqual(len(f.read()), size + 1) |
| 66 |
self.assertEqual(f.tell(), size + 1) |
| 67 |
|
| 68 |
def test_osstat(self): |
| 69 |
self.assertEqual(os.stat(TESTFN).st_size, size+1) |
| 70 |
|
| 71 |
def test_seek_read(self): |
| 72 |
with self.open(TESTFN, 'rb') as f: |
| 73 |
self.assertEqual(f.tell(), 0) |
| 74 |
self.assertEqual(f.read(1), b'z') |
| 75 |
self.assertEqual(f.tell(), 1) |
| 76 |
f.seek(0) |
| 77 |
self.assertEqual(f.tell(), 0) |
| 78 |
f.seek(0, 0) |
| 79 |
self.assertEqual(f.tell(), 0) |
| 80 |
f.seek(42) |
| 81 |
self.assertEqual(f.tell(), 42) |
| 82 |
f.seek(42, 0) |
| 83 |
self.assertEqual(f.tell(), 42) |
| 84 |
f.seek(42, 1) |
| 85 |
self.assertEqual(f.tell(), 84) |
| 86 |
f.seek(0, 1) |
| 87 |
self.assertEqual(f.tell(), 84) |
| 88 |
f.seek(0, 2) # seek from the end |
| 89 |
self.assertEqual(f.tell(), size + 1 + 0) |
| 90 |
f.seek(-10, 2) |
| 91 |
self.assertEqual(f.tell(), size + 1 - 10) |
| 92 |
f.seek(-size-1, 2) |
| 93 |
self.assertEqual(f.tell(), 0) |
| 94 |
f.seek(size) |
| 95 |
self.assertEqual(f.tell(), size) |
| 96 |
# the 'a' that was written at the end of file above |
| 97 |
self.assertEqual(f.read(1), b'a') |
| 98 |
f.seek(-size-1, 1) |
| 99 |
self.assertEqual(f.read(1), b'z') |
| 100 |
self.assertEqual(f.tell(), 1) |
| 101 |
|
| 102 |
def test_lseek(self): |
| 103 |
with self.open(TESTFN, 'rb') as f: |
| 104 |
self.assertEqual(os.lseek(f.fileno(), 0, 0), 0) |
| 105 |
self.assertEqual(os.lseek(f.fileno(), 42, 0), 42) |
| 106 |
self.assertEqual(os.lseek(f.fileno(), 42, 1), 84) |
| 107 |
self.assertEqual(os.lseek(f.fileno(), 0, 1), 84) |
| 108 |
self.assertEqual(os.lseek(f.fileno(), 0, 2), size+1+0) |
| 109 |
self.assertEqual(os.lseek(f.fileno(), -10, 2), size+1-10) |
| 110 |
self.assertEqual(os.lseek(f.fileno(), -size-1, 2), 0) |
| 111 |
self.assertEqual(os.lseek(f.fileno(), size, 0), size) |
| 112 |
# the 'a' that was written at the end of file above |
| 113 |
self.assertEqual(f.read(1), b'a') |
| 114 |
|
| 115 |
def test_truncate(self): |
| 116 |
with self.open(TESTFN, 'r+b') as f: |
| 117 |
if not hasattr(f, 'truncate'): |
| 118 |
raise unittest.SkipTest("open().truncate() not available " |
| 119 |
"on this system") |
| 120 |
f.seek(0, 2) |
| 121 |
# else we've lost track of the true size |
| 122 |
self.assertEqual(f.tell(), size+1) |
| 123 |
# Cut it back via seek + truncate with no argument. |
| 124 |
newsize = size - 10 |
| 125 |
f.seek(newsize) |
| 126 |
f.truncate() |
| 127 |
self.assertEqual(f.tell(), newsize) # else pointer moved |
| 128 |
f.seek(0, 2) |
| 129 |
self.assertEqual(f.tell(), newsize) # else wasn't truncated |
| 130 |
# Ensure that truncate(smaller than true size) shrinks |
| 131 |
# the file. |
| 132 |
newsize -= 1 |
| 133 |
f.seek(42) |
| 134 |
f.truncate(newsize) |
| 135 |
self.assertEqual(f.tell(), 42) |
| 136 |
f.seek(0, 2) |
| 137 |
self.assertEqual(f.tell(), newsize) |
| 138 |
# XXX truncate(larger than true size) is ill-defined |
| 139 |
# across platform; cut it waaaaay back |
| 140 |
f.seek(0) |
| 141 |
f.truncate(1) |
| 142 |
self.assertEqual(f.tell(), 0) # else pointer moved |
| 143 |
f.seek(0) |
| 144 |
# Verify readall on a truncated file is well behaved. read() |
| 145 |
# without a size can be unbounded, this should get just the byte |
| 146 |
# that remains. |
| 147 |
self.assertEqual(len(f.read()), 1) # else wasn't truncated |
| 148 |
|
| 149 |
def test_seekable(self): |
| 150 |
# Issue #5016; seekable() can return False when the current position |
| 151 |
# is negative when truncated to an int. |
| 152 |
for pos in (2**31-1, 2**31, 2**31+1): |
| 153 |
with self.open(TESTFN, 'rb') as f: |
| 154 |
f.seek(pos) |
| 155 |
self.assertTrue(f.seekable()) |
| 156 |
|
| 157 |
@bigmemtest(size=size, memuse=2, dry_run=False) |
| 158 |
def test_seek_readall(self, _size): |
| 159 |
# Seek which doesn't change position should readall successfully. |
| 160 |
with self.open(TESTFN, 'rb') as f: |
| 161 |
self.assertEqual(f.seek(0, os.SEEK_CUR), 0) |
| 162 |
self.assertEqual(len(f.read()), size + 1) |
| 163 |
|
| 164 |
# Seek which changes (or might change) position should readall |
| 165 |
# successfully. |
| 166 |
with self.open(TESTFN, 'rb') as f: |
| 167 |
self.assertEqual(f.seek(20, os.SEEK_SET), 20) |
| 168 |
self.assertEqual(len(f.read()), size - 19) |
| 169 |
|
| 170 |
with self.open(TESTFN, 'rb') as f: |
| 171 |
self.assertEqual(f.seek(-3, os.SEEK_END), size - 2) |
| 172 |
self.assertEqual(len(f.read()), 3) |
| 173 |
|
| 174 |
def skip_no_disk_space(path, required): |
| 175 |
def decorator(fun): |
| 176 |
def wrapper(*args, **kwargs): |
| 177 |
if not hasattr(shutil, "disk_usage"): |
| 178 |
raise unittest.SkipTest("requires shutil.disk_usage") |
| 179 |
if shutil.disk_usage(os.path.realpath(path)).free < required: |
| 180 |
hsize = int(required / 1024 / 1024) |
| 181 |
raise unittest.SkipTest( |
| 182 |
f"required {hsize} MiB of free disk space") |
| 183 |
return fun(*args, **kwargs) |
| 184 |
return wrapper |
| 185 |
return decorator |
| 186 |
|
| 187 |
|
| 188 |
class TestCopyfile(LargeFileTest, unittest.TestCase): |
| 189 |
open = staticmethod(io.open) |
| 190 |
|
| 191 |
# Exact required disk space would be (size * 2), but let's give it a |
| 192 |
# bit more tolerance. |
| 193 |
@skip_no_disk_space(TESTFN, size * 2.5) |
| 194 |
@requires_resource('cpu') |
| 195 |
def test_it(self): |
| 196 |
# Internally shutil.copyfile() can use "fast copy" methods like |
| 197 |
# os.sendfile(). |
| 198 |
size = os.path.getsize(TESTFN) |
| 199 |
shutil.copyfile(TESTFN, TESTFN2) |
| 200 |
self.assertEqual(os.path.getsize(TESTFN2), size) |
| 201 |
with open(TESTFN2, 'rb') as f: |
| 202 |
self.assertEqual(f.read(5), b'z\x00\x00\x00\x00') |
| 203 |
f.seek(size - 5) |
| 204 |
self.assertEqual(f.read(), b'\x00\x00\x00\x00a') |
| 205 |
|
| 206 |
|
| 207 |
@unittest.skipIf(not hasattr(os, 'sendfile'), 'sendfile not supported') |
| 208 |
class TestSocketSendfile(LargeFileTest, unittest.TestCase): |
| 209 |
open = staticmethod(io.open) |
| 210 |
timeout = SHORT_TIMEOUT |
| 211 |
|
| 212 |
def setUp(self): |
| 213 |
super().setUp() |
| 214 |
self.thread = None |
| 215 |
|
| 216 |
def tearDown(self): |
| 217 |
super().tearDown() |
| 218 |
if self.thread is not None: |
| 219 |
self.thread.join(self.timeout) |
| 220 |
self.thread = None |
| 221 |
|
| 222 |
def tcp_server(self, sock): |
| 223 |
def run(sock): |
| 224 |
with sock: |
| 225 |
conn, _ = sock.accept() |
| 226 |
conn.settimeout(self.timeout) |
| 227 |
with conn, open(TESTFN2, 'wb') as f: |
| 228 |
event.wait(self.timeout) |
| 229 |
while True: |
| 230 |
chunk = conn.recv(65536) |
| 231 |
if not chunk: |
| 232 |
return |
| 233 |
f.write(chunk) |
| 234 |
|
| 235 |
event = threading.Event() |
| 236 |
sock.settimeout(self.timeout) |
| 237 |
self.thread = threading.Thread(target=run, args=(sock, )) |
| 238 |
self.thread.start() |
| 239 |
event.set() |
| 240 |
|
| 241 |
# Exact required disk space would be (size * 2), but let's give it a |
| 242 |
# bit more tolerance. |
| 243 |
@skip_no_disk_space(TESTFN, size * 2.5) |
| 244 |
@requires_resource('cpu') |
| 245 |
def test_it(self): |
| 246 |
port = socket_helper.find_unused_port() |
| 247 |
with socket.create_server(("", port)) as sock: |
| 248 |
self.tcp_server(sock) |
| 249 |
with socket.create_connection(("127.0.0.1", port)) as client: |
| 250 |
with open(TESTFN, 'rb') as f: |
| 251 |
client.sendfile(f) |
| 252 |
self.tearDown() |
| 253 |
|
| 254 |
size = os.path.getsize(TESTFN) |
| 255 |
self.assertEqual(os.path.getsize(TESTFN2), size) |
| 256 |
with open(TESTFN2, 'rb') as f: |
| 257 |
self.assertEqual(f.read(5), b'z\x00\x00\x00\x00') |
| 258 |
f.seek(size - 5) |
| 259 |
self.assertEqual(f.read(), b'\x00\x00\x00\x00a') |
| 260 |
|
| 261 |
|
| 262 |
def setUpModule(): |
| 263 |
try: |
| 264 |
import signal |
| 265 |
# The default handler for SIGXFSZ is to abort the process. |
| 266 |
# By ignoring it, system calls exceeding the file size resource |
| 267 |
# limit will raise OSError instead of crashing the interpreter. |
| 268 |
signal.signal(signal.SIGXFSZ, signal.SIG_IGN) |
| 269 |
except (ImportError, AttributeError): |
| 270 |
pass |
| 271 |
|
| 272 |
# On Windows and Mac OSX this test consumes large resources; It |
| 273 |
# takes a long time to build the >2 GiB file and takes >2 GiB of disk |
| 274 |
# space therefore the resource must be enabled to run this test. |
| 275 |
# If not, nothing after this line stanza will be executed. |
| 276 |
if sys.platform[:3] == 'win' or sys.platform == 'darwin': |
| 277 |
requires('largefile', |
| 278 |
'test requires %s bytes and a long time to run' % str(size)) |
| 279 |
else: |
| 280 |
# Only run if the current filesystem supports large files. |
| 281 |
# (Skip this test on Windows, since we now always support |
| 282 |
# large files.) |
| 283 |
f = open(TESTFN, 'wb', buffering=0) |
| 284 |
try: |
| 285 |
# 2**31 == 2147483648 |
| 286 |
f.seek(2147483649) |
| 287 |
# Seeking is not enough of a test: you must write and flush, too! |
| 288 |
f.write(b'x') |
| 289 |
f.flush() |
| 290 |
except (OSError, OverflowError): |
| 291 |
raise unittest.SkipTest("filesystem does not have " |
| 292 |
"largefile support") |
| 293 |
finally: |
| 294 |
f.close() |
| 295 |
unlink(TESTFN) |
| 296 |
|
| 297 |
|
| 298 |
class CLargeFileTest(TestFileMethods, unittest.TestCase): |
| 299 |
open = staticmethod(io.open) |
| 300 |
|
| 301 |
|
| 302 |
class PyLargeFileTest(TestFileMethods, unittest.TestCase): |
| 303 |
open = staticmethod(pyio.open) |
| 304 |
|
| 305 |
|
| 306 |
def tearDownModule(): |
| 307 |
unlink(TESTFN) |
| 308 |
unlink(TESTFN2) |
| 309 |
|
| 310 |
|
| 311 |
if __name__ == '__main__': |
| 312 |
unittest.main() |