Debian Codegraph @

Lib/test/test_mmap.py

raw
1 from test.support import (
2 requires, _2G, _4G, gc_collect, cpython_only, is_emscripten, is_apple,
3 in_systemd_nspawn_sync_suppressed,
4 )
5 from test.support.import_helper import import_module
6 from test.support.os_helper import TESTFN, unlink
7 from test.support.script_helper import assert_python_ok
8 import unittest
9 import errno
10 import os
11 import re
12 import itertools
13 import random
14 import socket
15 import string
16 import sys
17 import textwrap
18 import weakref
19
20 # Skip test if we can't import mmap.
21 mmap = import_module('mmap')
22
23 PAGESIZE = mmap.PAGESIZE
24
25 tagname_prefix = f'python_{os.getpid()}_test_mmap'
26 def random_tagname(length=10):
27 suffix = ''.join(random.choices(string.ascii_uppercase, k=length))
28 return f'{tagname_prefix}_{suffix}'
29
30 # Python's mmap module dup()s the file descriptor. Emscripten's FS layer
31 # does not materialize file changes through a dupped fd to a new mmap.
32 if is_emscripten:
33 raise unittest.SkipTest("incompatible with Emscripten's mmap emulation.")
34
35
36 class MmapTests(unittest.TestCase):
37
38 def setUp(self):
39 if os.path.exists(TESTFN):
40 os.unlink(TESTFN)
41
42 def tearDown(self):
43 try:
44 os.unlink(TESTFN)
45 except OSError:
46 pass
47
48 def test_basic(self):
49 # Test mmap module on Unix systems and Windows
50
51 # Create a file to be mmap'ed.
52 f = open(TESTFN, 'bw+')
53 try:
54 # Write 2 pages worth of data to the file
55 f.write(b'\0'* PAGESIZE)
56 f.write(b'foo')
57 f.write(b'\0'* (PAGESIZE-3) )
58 f.flush()
59 m = mmap.mmap(f.fileno(), 2 * PAGESIZE)
60 finally:
61 f.close()
62
63 # Simple sanity checks
64
65 tp = str(type(m)) # SF bug 128713: segfaulted on Linux
66 self.assertEqual(m.find(b'foo'), PAGESIZE)
67
68 self.assertEqual(len(m), 2*PAGESIZE)
69
70 self.assertEqual(m[0], 0)
71 self.assertEqual(m[0:3], b'\0\0\0')
72
73 # Shouldn't crash on boundary (Issue #5292)
74 self.assertRaises(IndexError, m.__getitem__, len(m))
75 self.assertRaises(IndexError, m.__setitem__, len(m), b'\0')
76
77 # Modify the file's content
78 m[0] = b'3'[0]
79 m[PAGESIZE +3: PAGESIZE +3+3] = b'bar'
80
81 # Check that the modification worked
82 self.assertEqual(m[0], b'3'[0])
83 self.assertEqual(m[0:3], b'3\0\0')
84 self.assertEqual(m[PAGESIZE-1 : PAGESIZE + 7], b'\0foobar\0')
85
86 m.flush()
87
88 # Test doing a regular expression match in an mmap'ed file
89 match = re.search(b'[A-Za-z]+', m)
90 if match is None:
91 self.fail('regex match on mmap failed!')
92 else:
93 start, end = match.span(0)
94 length = end - start
95
96 self.assertEqual(start, PAGESIZE)
97 self.assertEqual(end, PAGESIZE + 6)
98
99 # test seeking around (try to overflow the seek implementation)
100 self.assertTrue(m.seekable())
101 self.assertEqual(m.seek(0, 0), 0)
102 self.assertEqual(m.tell(), 0)
103 self.assertEqual(m.seek(42, 1), 42)
104 self.assertEqual(m.tell(), 42)
105 self.assertEqual(m.seek(0, 2), len(m))
106 self.assertEqual(m.tell(), len(m))
107
108 # Try to seek to negative position...
109 self.assertRaises(ValueError, m.seek, -1)
110
111 # Try to seek beyond end of mmap...
112 self.assertRaises(ValueError, m.seek, 1, 2)
113
114 # Try to seek to negative position...
115 self.assertRaises(ValueError, m.seek, -len(m)-1, 2)
116
117 # Try resizing map
118 try:
119 m.resize(512)
120 except SystemError:
121 # resize() not supported
122 # No messages are printed, since the output of this test suite
123 # would then be different across platforms.
124 pass
125 else:
126 # resize() is supported
127 self.assertEqual(len(m), 512)
128 # Check that we can no longer seek beyond the new size.
129 self.assertRaises(ValueError, m.seek, 513, 0)
130
131 # Check that the underlying file is truncated too
132 # (bug #728515)
133 f = open(TESTFN, 'rb')
134 try:
135 f.seek(0, 2)
136 self.assertEqual(f.tell(), 512)
137 finally:
138 f.close()
139 self.assertEqual(m.size(), 512)
140
141 m.close()
142
143 def test_access_parameter(self):
144 # Test for "access" keyword parameter
145 mapsize = 10
146 with open(TESTFN, "wb") as fp:
147 fp.write(b"a"*mapsize)
148 with open(TESTFN, "rb") as f:
149 m = mmap.mmap(f.fileno(), mapsize, access=mmap.ACCESS_READ)
150 self.assertEqual(m[:], b'a'*mapsize, "Readonly memory map data incorrect.")
151
152 # Ensuring that readonly mmap can't be slice assigned
153 try:
154 m[:] = b'b'*mapsize
155 except TypeError:
156 pass
157 else:
158 self.fail("Able to write to readonly memory map")
159
160 # Ensuring that readonly mmap can't be item assigned
161 try:
162 m[0] = b'b'
163 except TypeError:
164 pass
165 else:
166 self.fail("Able to write to readonly memory map")
167
168 # Ensuring that readonly mmap can't be write() to
169 try:
170 m.seek(0, 0)
171 m.write(b'abc')
172 except TypeError:
173 pass
174 else:
175 self.fail("Able to write to readonly memory map")
176
177 # Ensuring that readonly mmap can't be write_byte() to
178 try:
179 m.seek(0, 0)
180 m.write_byte(b'd')
181 except TypeError:
182 pass
183 else:
184 self.fail("Able to write to readonly memory map")
185
186 # Ensuring that readonly mmap can't be resized
187 try:
188 m.resize(2*mapsize)
189 except SystemError: # resize is not universally supported
190 pass
191 except TypeError:
192 pass
193 else:
194 self.fail("Able to resize readonly memory map")
195 with open(TESTFN, "rb") as fp:
196 self.assertEqual(fp.read(), b'a'*mapsize,
197 "Readonly memory map data file was modified")
198
199 # Opening mmap with size too big
200 with open(TESTFN, "r+b") as f:
201 try:
202 m = mmap.mmap(f.fileno(), mapsize+1)
203 except ValueError:
204 # we do not expect a ValueError on Windows
205 # CAUTION: This also changes the size of the file on disk, and
206 # later tests assume that the length hasn't changed. We need to
207 # repair that.
208 if sys.platform.startswith('win'):
209 self.fail("Opening mmap with size+1 should work on Windows.")
210 else:
211 # we expect a ValueError on Unix, but not on Windows
212 if not sys.platform.startswith('win'):
213 self.fail("Opening mmap with size+1 should raise ValueError.")
214 m.close()
215 if sys.platform.startswith('win'):
216 # Repair damage from the resizing test.
217 with open(TESTFN, 'r+b') as f:
218 f.truncate(mapsize)
219
220 # Opening mmap with access=ACCESS_WRITE
221 with open(TESTFN, "r+b") as f:
222 m = mmap.mmap(f.fileno(), mapsize, access=mmap.ACCESS_WRITE)
223 # Modifying write-through memory map
224 m[:] = b'c'*mapsize
225 self.assertEqual(m[:], b'c'*mapsize,
226 "Write-through memory map memory not updated properly.")
227 m.flush()
228 m.close()
229 with open(TESTFN, 'rb') as f:
230 stuff = f.read()
231 self.assertEqual(stuff, b'c'*mapsize,
232 "Write-through memory map data file not updated properly.")
233
234 # Opening mmap with access=ACCESS_COPY
235 with open(TESTFN, "r+b") as f:
236 m = mmap.mmap(f.fileno(), mapsize, access=mmap.ACCESS_COPY)
237 # Modifying copy-on-write memory map
238 m[:] = b'd'*mapsize
239 self.assertEqual(m[:], b'd' * mapsize,
240 "Copy-on-write memory map data not written correctly.")
241 m.flush()
242 with open(TESTFN, "rb") as fp:
243 self.assertEqual(fp.read(), b'c'*mapsize,
244 "Copy-on-write test data file should not be modified.")
245 # Ensuring copy-on-write maps cannot be resized
246 self.assertRaises(TypeError, m.resize, 2*mapsize)
247 m.close()
248
249 # Ensuring invalid access parameter raises exception
250 with open(TESTFN, "r+b") as f:
251 self.assertRaises(ValueError, mmap.mmap, f.fileno(), mapsize, access=4)
252
253 if os.name == "posix":
254 # Try incompatible flags, prot and access parameters.
255 with open(TESTFN, "r+b") as f:
256 self.assertRaises(ValueError, mmap.mmap, f.fileno(), mapsize,
257 flags=mmap.MAP_PRIVATE,
258 prot=mmap.PROT_READ, access=mmap.ACCESS_WRITE)
259
260 # Try writing with PROT_EXEC and without PROT_WRITE
261 prot = mmap.PROT_READ | getattr(mmap, 'PROT_EXEC', 0)
262 with open(TESTFN, "r+b") as f:
263 try:
264 m = mmap.mmap(f.fileno(), mapsize, prot=prot)
265 except PermissionError:
266 # on macOS 14, PROT_READ | PROT_EXEC is not allowed
267 pass
268 else:
269 self.assertRaises(TypeError, m.write, b"abcdef")
270 self.assertRaises(TypeError, m.write_byte, 0)
271 m.close()
272
273 @unittest.skipIf(os.name == 'nt', 'trackfd not present on Windows')
274 def test_trackfd_parameter(self):
275 size = 64
276 with open(TESTFN, "wb") as f:
277 f.write(b"a"*size)
278 for close_original_fd in True, False:
279 with self.subTest(close_original_fd=close_original_fd):
280 with open(TESTFN, "r+b") as f:
281 with mmap.mmap(f.fileno(), size, trackfd=False) as m:
282 if close_original_fd:
283 f.close()
284 self.assertEqual(len(m), size)
285 with self.assertRaises(OSError) as err_cm:
286 m.size()
287 self.assertEqual(err_cm.exception.errno, errno.EBADF)
288 with self.assertRaises(ValueError):
289 m.resize(size * 2)
290 with self.assertRaises(ValueError):
291 m.resize(size // 2)
292 self.assertEqual(m.closed, False)
293
294 # Smoke-test other API
295 m.write_byte(ord('X'))
296 m[2] = ord('Y')
297 m.flush()
298 with open(TESTFN, "rb") as f:
299 self.assertEqual(f.read(4), b'XaYa')
300 self.assertEqual(m.tell(), 1)
301 m.seek(0)
302 self.assertEqual(m.tell(), 0)
303 self.assertEqual(m.read_byte(), ord('X'))
304
305 self.assertEqual(m.closed, True)
306 self.assertEqual(os.stat(TESTFN).st_size, size)
307
308 @unittest.skipIf(os.name == 'nt', 'trackfd not present on Windows')
309 def test_trackfd_neg1(self):
310 size = 64
311 with mmap.mmap(-1, size, trackfd=False) as m:
312 with self.assertRaises(OSError):
313 m.size()
314 with self.assertRaises(ValueError):
315 m.resize(size // 2)
316 self.assertEqual(len(m), size)
317 m[0] = ord('a')
318 assert m[0] == ord('a')
319
320 @unittest.skipIf(os.name != 'nt', 'trackfd only fails on Windows')
321 def test_no_trackfd_parameter_on_windows(self):
322 # 'trackffd' is an invalid keyword argument for this function
323 size = 64
324 with self.assertRaises(TypeError):
325 mmap.mmap(-1, size, trackfd=True)
326 with self.assertRaises(TypeError):
327 mmap.mmap(-1, size, trackfd=False)
328
329 def test_bad_file_desc(self):
330 # Try opening a bad file descriptor...
331 self.assertRaises(OSError, mmap.mmap, -2, 4096)
332
333 def test_tougher_find(self):
334 # Do a tougher .find() test. SF bug 515943 pointed out that, in 2.2,
335 # searching for data with embedded \0 bytes didn't work.
336 with open(TESTFN, 'wb+') as f:
337
338 data = b'aabaac\x00deef\x00\x00aa\x00'
339 n = len(data)
340 f.write(data)
341 f.flush()
342 m = mmap.mmap(f.fileno(), n)
343
344 for start in range(n+1):
345 for finish in range(start, n+1):
346 slice = data[start : finish]
347 self.assertEqual(m.find(slice), data.find(slice))
348 self.assertEqual(m.find(slice + b'x'), -1)
349 m.close()
350
351 def test_find_end(self):
352 # test the new 'end' parameter works as expected
353 with open(TESTFN, 'wb+') as f:
354 data = b'one two ones'
355 n = len(data)
356 f.write(data)
357 f.flush()
358 m = mmap.mmap(f.fileno(), n)
359
360 self.assertEqual(m.find(b'one'), 0)
361 self.assertEqual(m.find(b'ones'), 8)
362 self.assertEqual(m.find(b'one', 0, -1), 0)
363 self.assertEqual(m.find(b'one', 1), 8)
364 self.assertEqual(m.find(b'one', 1, -1), 8)
365 self.assertEqual(m.find(b'one', 1, -2), -1)
366 self.assertEqual(m.find(bytearray(b'one')), 0)
367
368 for i in range(-n-1, n+1):
369 for j in range(-n-1, n+1):
370 for p in [b"o", b"on", b"two", b"ones", b"s"]:
371 expected = data.find(p, i, j)
372 self.assertEqual(m.find(p, i, j), expected, (p, i, j))
373
374 def test_find_does_not_access_beyond_buffer(self):
375 try:
376 flags = mmap.MAP_PRIVATE | mmap.MAP_ANONYMOUS
377 PAGESIZE = mmap.PAGESIZE
378 PROT_NONE = 0
379 PROT_READ = mmap.PROT_READ
380 except AttributeError as e:
381 raise unittest.SkipTest("mmap flags unavailable") from e
382 for i in range(0, 2049):
383 with mmap.mmap(-1, PAGESIZE * (i + 1),
384 flags=flags, prot=PROT_NONE) as guard:
385 with mmap.mmap(-1, PAGESIZE * (i + 2048),
386 flags=flags, prot=PROT_READ) as fm:
387 fm.find(b"fo", -2)
388
389
390 def test_rfind(self):
391 # test the new 'end' parameter works as expected
392 with open(TESTFN, 'wb+') as f:
393 data = b'one two ones'
394 n = len(data)
395 f.write(data)
396 f.flush()
397 m = mmap.mmap(f.fileno(), n)
398
399 self.assertEqual(m.rfind(b'one'), 8)
400 self.assertEqual(m.rfind(b'one '), 0)
401 self.assertEqual(m.rfind(b'one', 0, -1), 8)
402 self.assertEqual(m.rfind(b'one', 0, -2), 0)
403 self.assertEqual(m.rfind(b'one', 1, -1), 8)
404 self.assertEqual(m.rfind(b'one', 1, -2), -1)
405 self.assertEqual(m.rfind(bytearray(b'one')), 8)
406
407
408 def test_double_close(self):
409 # make sure a double close doesn't crash on Solaris (Bug# 665913)
410 with open(TESTFN, 'wb+') as f:
411 f.write(2**16 * b'a') # Arbitrary character
412
413 with open(TESTFN, 'rb') as f:
414 mf = mmap.mmap(f.fileno(), 2**16, access=mmap.ACCESS_READ)
415 mf.close()
416 mf.close()
417
418 def test_entire_file(self):
419 # test mapping of entire file by passing 0 for map length
420 with open(TESTFN, "wb+") as f:
421 f.write(2**16 * b'm') # Arbitrary character
422
423 with open(TESTFN, "rb+") as f, \
424 mmap.mmap(f.fileno(), 0) as mf:
425 self.assertEqual(len(mf), 2**16, "Map size should equal file size.")
426 self.assertEqual(mf.read(2**16), 2**16 * b"m")
427
428 def test_length_0_offset(self):
429 # Issue #10916: test mapping of remainder of file by passing 0 for
430 # map length with an offset doesn't cause a segfault.
431 # NOTE: allocation granularity is currently 65536 under Win64,
432 # and therefore the minimum offset alignment.
433 with open(TESTFN, "wb") as f:
434 f.write((65536 * 2) * b'm') # Arbitrary character
435
436 with open(TESTFN, "rb") as f:
437 with mmap.mmap(f.fileno(), 0, offset=65536, access=mmap.ACCESS_READ) as mf:
438 self.assertRaises(IndexError, mf.__getitem__, 80000)
439
440 def test_length_0_large_offset(self):
441 # Issue #10959: test mapping of a file by passing 0 for
442 # map length with a large offset doesn't cause a segfault.
443 with open(TESTFN, "wb") as f:
444 f.write(115699 * b'm') # Arbitrary character
445
446 with open(TESTFN, "w+b") as f:
447 self.assertRaises(ValueError, mmap.mmap, f.fileno(), 0,
448 offset=2147418112)
449
450 def test_move(self):
451 # make move works everywhere (64-bit format problem earlier)
452 with open(TESTFN, 'wb+') as f:
453
454 f.write(b"ABCDEabcde") # Arbitrary character
455 f.flush()
456
457 mf = mmap.mmap(f.fileno(), 10)
458 mf.move(5, 0, 5)
459 self.assertEqual(mf[:], b"ABCDEABCDE", "Map move should have duplicated front 5")
460 mf.close()
461
462 # more excessive test
463 data = b"0123456789"
464 for dest in range(len(data)):
465 for src in range(len(data)):
466 for count in range(len(data) - max(dest, src)):
467 expected = data[:dest] + data[src:src+count] + data[dest+count:]
468 m = mmap.mmap(-1, len(data))
469 m[:] = data
470 m.move(dest, src, count)
471 self.assertEqual(m[:], expected)
472 m.close()
473
474 # segfault test (Issue 5387)
475 m = mmap.mmap(-1, 100)
476 offsets = [-100, -1, 0, 1, 100]
477 for source, dest, size in itertools.product(offsets, offsets, offsets):
478 try:
479 m.move(source, dest, size)
480 except ValueError:
481 pass
482
483 offsets = [(-1, -1, -1), (-1, -1, 0), (-1, 0, -1), (0, -1, -1),
484 (-1, 0, 0), (0, -1, 0), (0, 0, -1)]
485 for source, dest, size in offsets:
486 self.assertRaises(ValueError, m.move, source, dest, size)
487
488 m.close()
489
490 m = mmap.mmap(-1, 1) # single byte
491 self.assertRaises(ValueError, m.move, 0, 0, 2)
492 self.assertRaises(ValueError, m.move, 1, 0, 1)
493 self.assertRaises(ValueError, m.move, 0, 1, 1)
494 m.move(0, 0, 1)
495 m.move(0, 0, 0)
496
497 def test_anonymous(self):
498 # anonymous mmap.mmap(-1, PAGE)
499 m = mmap.mmap(-1, PAGESIZE)
500 for x in range(PAGESIZE):
501 self.assertEqual(m[x], 0,
502 "anonymously mmap'ed contents should be zero")
503
504 for x in range(PAGESIZE):
505 b = x & 0xff
506 m[x] = b
507 self.assertEqual(m[x], b)
508
509 def test_read_all(self):
510 m = mmap.mmap(-1, 16)
511 self.addCleanup(m.close)
512
513 # With no parameters, or None or a negative argument, reads all
514 m.write(bytes(range(16)))
515 m.seek(0)
516 self.assertEqual(m.read(), bytes(range(16)))
517 m.seek(8)
518 self.assertEqual(m.read(), bytes(range(8, 16)))
519 m.seek(16)
520 self.assertEqual(m.read(), b'')
521 m.seek(3)
522 self.assertEqual(m.read(None), bytes(range(3, 16)))
523 m.seek(4)
524 self.assertEqual(m.read(-1), bytes(range(4, 16)))
525 m.seek(5)
526 self.assertEqual(m.read(-2), bytes(range(5, 16)))
527 m.seek(9)
528 self.assertEqual(m.read(-42), bytes(range(9, 16)))
529
530 def test_read_invalid_arg(self):
531 m = mmap.mmap(-1, 16)
532 self.addCleanup(m.close)
533
534 self.assertRaises(TypeError, m.read, 'foo')
535 self.assertRaises(TypeError, m.read, 5.5)
536 self.assertRaises(TypeError, m.read, [1, 2, 3])
537
538 def test_extended_getslice(self):
539 # Test extended slicing by comparing with list slicing.
540 s = bytes(reversed(range(256)))
541 m = mmap.mmap(-1, len(s))
542 m[:] = s
543 self.assertEqual(m[:], s)
544 indices = (0, None, 1, 3, 19, 300, sys.maxsize, -1, -2, -31, -300)
545 for start in indices:
546 for stop in indices:
547 # Skip step 0 (invalid)
548 for step in indices[1:]:
549 self.assertEqual(m[start:stop:step],
550 s[start:stop:step])
551
552 def test_extended_set_del_slice(self):
553 # Test extended slicing by comparing with list slicing.
554 s = bytes(reversed(range(256)))
555 m = mmap.mmap(-1, len(s))
556 indices = (0, None, 1, 3, 19, 300, sys.maxsize, -1, -2, -31, -300)
557 for start in indices:
558 for stop in indices:
559 # Skip invalid step 0
560 for step in indices[1:]:
561 m[:] = s
562 self.assertEqual(m[:], s)
563 L = list(s)
564 # Make sure we have a slice of exactly the right length,
565 # but with different data.
566 data = L[start:stop:step]
567 data = bytes(reversed(data))
568 L[start:stop:step] = data
569 m[start:stop:step] = data
570 self.assertEqual(m[:], bytes(L))
571
572 def make_mmap_file (self, f, halfsize):
573 # Write 2 pages worth of data to the file
574 f.write (b'\0' * halfsize)
575 f.write (b'foo')
576 f.write (b'\0' * (halfsize - 3))
577 f.flush ()
578 return mmap.mmap (f.fileno(), 0)
579
580 def test_empty_file (self):
581 f = open (TESTFN, 'w+b')
582 f.close()
583 with open(TESTFN, "rb") as f :
584 self.assertRaisesRegex(ValueError,
585 "cannot mmap an empty file",
586 mmap.mmap, f.fileno(), 0,
587 access=mmap.ACCESS_READ)
588
589 def test_offset (self):
590 f = open (TESTFN, 'w+b')
591
592 try: # unlink TESTFN no matter what
593 halfsize = mmap.ALLOCATIONGRANULARITY
594 m = self.make_mmap_file (f, halfsize)
595 m.close ()
596 f.close ()
597
598 mapsize = halfsize * 2
599 # Try invalid offset
600 f = open(TESTFN, "r+b")
601 for offset in [-2, -1, None]:
602 try:
603 m = mmap.mmap(f.fileno(), mapsize, offset=offset)
604 self.assertEqual(0, 1)
605 except (ValueError, TypeError, OverflowError):
606 pass
607 else:
608 self.assertEqual(0, 0)
609 f.close()
610
611 # Try valid offset, hopefully 8192 works on all OSes
612 f = open(TESTFN, "r+b")
613 m = mmap.mmap(f.fileno(), mapsize - halfsize, offset=halfsize)
614 self.assertEqual(m[0:3], b'foo')
615 f.close()
616
617 # Try resizing map
618 try:
619 m.resize(512)
620 except SystemError:
621 pass
622 else:
623 # resize() is supported
624 self.assertEqual(len(m), 512)
625 # Check that we can no longer seek beyond the new size.
626 self.assertRaises(ValueError, m.seek, 513, 0)
627 # Check that the content is not changed
628 self.assertEqual(m[0:3], b'foo')
629
630 # Check that the underlying file is truncated too
631 f = open(TESTFN, 'rb')
632 f.seek(0, 2)
633 self.assertEqual(f.tell(), halfsize + 512)
634 f.close()
635 self.assertEqual(m.size(), halfsize + 512)
636
637 m.close()
638
639 finally:
640 f.close()
641 try:
642 os.unlink(TESTFN)
643 except OSError:
644 pass
645
646 def test_subclass(self):
647 class anon_mmap(mmap.mmap):
648 def __new__(klass, *args, **kwargs):
649 return mmap.mmap.__new__(klass, -1, *args, **kwargs)
650 anon_mmap(PAGESIZE)
651
652 @unittest.skipUnless(hasattr(mmap, 'PROT_READ'), "needs mmap.PROT_READ")
653 def test_prot_readonly(self):
654 mapsize = 10
655 with open(TESTFN, "wb") as fp:
656 fp.write(b"a"*mapsize)
657 with open(TESTFN, "rb") as f:
658 m = mmap.mmap(f.fileno(), mapsize, prot=mmap.PROT_READ)
659 self.assertRaises(TypeError, m.write, "foo")
660
661 def test_error(self):
662 self.assertIs(mmap.error, OSError)
663
664 def test_io_methods(self):
665 data = b"0123456789"
666 with open(TESTFN, "wb") as fp:
667 fp.write(b"x"*len(data))
668 with open(TESTFN, "r+b") as f:
669 m = mmap.mmap(f.fileno(), len(data))
670 # Test write_byte()
671 for i in range(len(data)):
672 self.assertEqual(m.tell(), i)
673 m.write_byte(data[i])
674 self.assertEqual(m.tell(), i+1)
675 self.assertRaises(ValueError, m.write_byte, b"x"[0])
676 self.assertEqual(m[:], data)
677 # Test read_byte()
678 m.seek(0)
679 for i in range(len(data)):
680 self.assertEqual(m.tell(), i)
681 self.assertEqual(m.read_byte(), data[i])
682 self.assertEqual(m.tell(), i+1)
683 self.assertRaises(ValueError, m.read_byte)
684 # Test read()
685 m.seek(3)
686 self.assertEqual(m.read(3), b"345")
687 self.assertEqual(m.tell(), 6)
688 # Test write()
689 m.seek(3)
690 m.write(b"bar")
691 self.assertEqual(m.tell(), 6)
692 self.assertEqual(m[:], b"012bar6789")
693 m.write(bytearray(b"baz"))
694 self.assertEqual(m.tell(), 9)
695 self.assertEqual(m[:], b"012barbaz9")
696 self.assertRaises(ValueError, m.write, b"ba")
697
698 def test_non_ascii_byte(self):
699 for b in (129, 200, 255): # > 128
700 m = mmap.mmap(-1, 1)
701 m.write_byte(b)
702 self.assertEqual(m[0], b)
703 m.seek(0)
704 self.assertEqual(m.read_byte(), b)
705 m.close()
706
707 @unittest.skipUnless(os.name == 'nt', 'requires Windows')
708 def test_tagname(self):
709 data1 = b"0123456789"
710 data2 = b"abcdefghij"
711 assert len(data1) == len(data2)
712 tagname1 = random_tagname()
713 tagname2 = random_tagname()
714
715 # Test same tag
716 m1 = mmap.mmap(-1, len(data1), tagname=tagname1)
717 m1[:] = data1
718 m2 = mmap.mmap(-1, len(data2), tagname=tagname1)
719 m2[:] = data2
720 self.assertEqual(m1[:], data2)
721 self.assertEqual(m2[:], data2)
722 m2.close()
723 m1.close()
724
725 # Test different tag
726 m1 = mmap.mmap(-1, len(data1), tagname=tagname1)
727 m1[:] = data1
728 m2 = mmap.mmap(-1, len(data2), tagname=tagname2)
729 m2[:] = data2
730 self.assertEqual(m1[:], data1)
731 self.assertEqual(m2[:], data2)
732 m2.close()
733 m1.close()
734
735 with self.assertRaisesRegex(TypeError, 'tagname'):
736 mmap.mmap(-1, 8, tagname=1)
737
738 @cpython_only
739 @unittest.skipUnless(os.name == 'nt', 'requires Windows')
740 def test_sizeof(self):
741 m1 = mmap.mmap(-1, 100)
742 tagname = random_tagname()
743 m2 = mmap.mmap(-1, 100, tagname=tagname)
744 self.assertGreater(sys.getsizeof(m2), sys.getsizeof(m1))
745
746 @unittest.skipUnless(os.name == 'nt', 'requires Windows')
747 def test_crasher_on_windows(self):
748 # Should not crash (Issue 1733986)
749 tagname = random_tagname()
750 m = mmap.mmap(-1, 1000, tagname=tagname)
751 try:
752 mmap.mmap(-1, 5000, tagname=tagname)[:] # same tagname, but larger size
753 except:
754 pass
755 m.close()
756
757 # Should not crash (Issue 5385)
758 with open(TESTFN, "wb") as fp:
759 fp.write(b"x"*10)
760 f = open(TESTFN, "r+b")
761 m = mmap.mmap(f.fileno(), 0)
762 f.close()
763 try:
764 m.resize(0) # will raise OSError
765 except:
766 pass
767 try:
768 m[:]
769 except:
770 pass
771 m.close()
772
773 @unittest.skipUnless(os.name == 'nt', 'requires Windows')
774 def test_invalid_descriptor(self):
775 # socket file descriptors are valid, but out of range
776 # for _get_osfhandle, causing a crash when validating the
777 # parameters to _get_osfhandle.
778 s = socket.socket()
779 try:
780 with self.assertRaises(OSError):
781 m = mmap.mmap(s.fileno(), 10)
782 finally:
783 s.close()
784
785 def test_context_manager(self):
786 with mmap.mmap(-1, 10) as m:
787 self.assertFalse(m.closed)
788 self.assertTrue(m.closed)
789
790 def test_context_manager_exception(self):
791 # Test that the OSError gets passed through
792 with self.assertRaises(Exception) as exc:
793 with mmap.mmap(-1, 10) as m:
794 raise OSError
795 self.assertIsInstance(exc.exception, OSError,
796 "wrong exception raised in context manager")
797 self.assertTrue(m.closed, "context manager failed")
798
799 def test_weakref(self):
800 # Check mmap objects are weakrefable
801 mm = mmap.mmap(-1, 16)
802 wr = weakref.ref(mm)
803 self.assertIs(wr(), mm)
804 del mm
805 gc_collect()
806 self.assertIs(wr(), None)
807
808 def test_write_returning_the_number_of_bytes_written(self):
809 mm = mmap.mmap(-1, 16)
810 self.assertEqual(mm.write(b""), 0)
811 self.assertEqual(mm.write(b"x"), 1)
812 self.assertEqual(mm.write(b"yz"), 2)
813 self.assertEqual(mm.write(b"python"), 6)
814
815 def test_resize_past_pos(self):
816 m = mmap.mmap(-1, 8192)
817 self.addCleanup(m.close)
818 m.read(5000)
819 try:
820 m.resize(4096)
821 except SystemError:
822 self.skipTest("resizing not supported")
823 self.assertEqual(m.read(14), b'')
824 self.assertRaises(ValueError, m.read_byte)
825 self.assertRaises(ValueError, m.write_byte, 42)
826 self.assertRaises(ValueError, m.write, b'abc')
827
828 def test_concat_repeat_exception(self):
829 m = mmap.mmap(-1, 16)
830 with self.assertRaises(TypeError):
831 m + m
832 with self.assertRaises(TypeError):
833 m * 2
834
835 def test_flush_return_value(self):
836 # mm.flush() should return None on success, raise an
837 # exception on error under all platforms.
838 mm = mmap.mmap(-1, 16)
839 self.addCleanup(mm.close)
840 mm.write(b'python')
841 result = mm.flush()
842 self.assertIsNone(result)
843 if (sys.platform.startswith(('linux', 'android'))
844 and not in_systemd_nspawn_sync_suppressed()):
845 # 'offset' must be a multiple of mmap.PAGESIZE on Linux.
846 # See bpo-34754 for details.
847 self.assertRaises(OSError, mm.flush, 1, len(b'python'))
848
849 def test_repr(self):
850 open_mmap_repr_pat = re.compile(
851 r"<mmap.mmap closed=False, "
852 r"access=(?P<access>\S+), "
853 r"length=(?P<length>\d+), "
854 r"pos=(?P<pos>\d+), "
855 r"offset=(?P<offset>\d+)>")
856 closed_mmap_repr_pat = re.compile(r"<mmap.mmap closed=True>")
857 mapsizes = (50, 100, 1_000, 1_000_000, 10_000_000)
858 offsets = tuple((mapsize // 2 // mmap.ALLOCATIONGRANULARITY)
859 * mmap.ALLOCATIONGRANULARITY for mapsize in mapsizes)
860 for offset, mapsize in zip(offsets, mapsizes):
861 data = b'a' * mapsize
862 length = mapsize - offset
863 accesses = ('ACCESS_DEFAULT', 'ACCESS_READ',
864 'ACCESS_COPY', 'ACCESS_WRITE')
865 positions = (0, length//10, length//5, length//4)
866 with open(TESTFN, "wb+") as fp:
867 fp.write(data)
868 fp.flush()
869 for access, pos in itertools.product(accesses, positions):
870 accint = getattr(mmap, access)
871 with mmap.mmap(fp.fileno(),
872 length,
873 access=accint,
874 offset=offset) as mm:
875 mm.seek(pos)
876 match = open_mmap_repr_pat.match(repr(mm))
877 self.assertIsNotNone(match)
878 self.assertEqual(match.group('access'), access)
879 self.assertEqual(match.group('length'), str(length))
880 self.assertEqual(match.group('pos'), str(pos))
881 self.assertEqual(match.group('offset'), str(offset))
882 match = closed_mmap_repr_pat.match(repr(mm))
883 self.assertIsNotNone(match)
884
885 @unittest.skipUnless(hasattr(mmap.mmap, 'madvise'), 'needs madvise')
886 def test_madvise(self):
887 size = 2 * PAGESIZE
888 m = mmap.mmap(-1, size)
889
890 class Number:
891 def __index__(self):
892 return 2
893
894 with self.assertRaisesRegex(ValueError, "madvise start out of bounds"):
895 m.madvise(mmap.MADV_NORMAL, size)
896 with self.assertRaisesRegex(ValueError, "madvise start out of bounds"):
897 m.madvise(mmap.MADV_NORMAL, -1)
898 with self.assertRaisesRegex(ValueError, "madvise length invalid"):
899 m.madvise(mmap.MADV_NORMAL, 0, -1)
900 with self.assertRaisesRegex(OverflowError, "madvise length too large"):
901 m.madvise(mmap.MADV_NORMAL, PAGESIZE, sys.maxsize)
902 with self.assertRaisesRegex(
903 TypeError, "'str' object cannot be interpreted as an integer"):
904 m.madvise(mmap.MADV_NORMAL, PAGESIZE, "Not a Number")
905 self.assertEqual(m.madvise(mmap.MADV_NORMAL), None)
906 self.assertEqual(m.madvise(mmap.MADV_NORMAL, PAGESIZE), None)
907 self.assertEqual(m.madvise(mmap.MADV_NORMAL, PAGESIZE, size), None)
908 self.assertEqual(m.madvise(mmap.MADV_NORMAL, 0, 2), None)
909 self.assertEqual(m.madvise(mmap.MADV_NORMAL, 0, Number()), None)
910 self.assertEqual(m.madvise(mmap.MADV_NORMAL, 0, size), None)
911
912 def test_resize_up_anonymous_mapping(self):
913 """If the mmap is backed by the pagefile ensure a resize up can happen
914 and that the original data is still in place
915 """
916 start_size = PAGESIZE
917 new_size = 2 * start_size
918 data = random.randbytes(start_size)
919
920 with mmap.mmap(-1, start_size) as m:
921 m[:] = data
922 if sys.platform.startswith(('linux', 'android')):
923 # Can't expand a shared anonymous mapping on Linux.
924 # See https://bugzilla.kernel.org/show_bug.cgi?id=8691
925 with self.assertRaises(ValueError):
926 m.resize(new_size)
927 else:
928 try:
929 m.resize(new_size)
930 except SystemError:
931 pass
932 else:
933 self.assertEqual(len(m), new_size)
934 self.assertEqual(m[:start_size], data)
935 self.assertEqual(m[start_size:], b'\0' * (new_size - start_size))
936
937 @unittest.skipUnless(os.name == 'posix', 'requires Posix')
938 def test_resize_up_private_anonymous_mapping(self):
939 start_size = PAGESIZE
940 new_size = 2 * start_size
941 data = random.randbytes(start_size)
942
943 with mmap.mmap(-1, start_size, flags=mmap.MAP_PRIVATE) as m:
944 m[:] = data
945 try:
946 m.resize(new_size)
947 except SystemError:
948 pass
949 else:
950 self.assertEqual(len(m), new_size)
951 self.assertEqual(m[:start_size], data)
952 self.assertEqual(m[start_size:], b'\0' * (new_size - start_size))
953
954 def test_resize_down_anonymous_mapping(self):
955 """If the mmap is backed by the pagefile ensure a resize down up can happen
956 and that a truncated form of the original data is still in place
957 """
958 start_size = 2 * PAGESIZE
959 new_size = start_size // 2
960 data = random.randbytes(start_size)
961
962 with mmap.mmap(-1, start_size) as m:
963 m[:] = data
964 try:
965 m.resize(new_size)
966 except SystemError:
967 pass
968 else:
969 self.assertEqual(len(m), new_size)
970 self.assertEqual(m[:], data[:new_size])
971 if sys.platform.startswith(('linux', 'android')):
972 # Can't expand to its original size.
973 with self.assertRaises(ValueError):
974 m.resize(start_size)
975
976 @unittest.skipUnless(os.name == 'nt', 'requires Windows')
977 def test_resize_fails_if_mapping_held_elsewhere(self):
978 """If more than one mapping is held against a named file on Windows, neither
979 mapping can be resized
980 """
981 start_size = 2 * PAGESIZE
982 reduced_size = PAGESIZE
983
984 f = open(TESTFN, 'wb+')
985 f.truncate(start_size)
986 try:
987 m1 = mmap.mmap(f.fileno(), start_size)
988 m2 = mmap.mmap(f.fileno(), start_size)
989 with self.assertRaises(OSError):
990 m1.resize(reduced_size)
991 with self.assertRaises(OSError):
992 m2.resize(reduced_size)
993 m2.close()
994 m1.resize(reduced_size)
995 self.assertEqual(m1.size(), reduced_size)
996 self.assertEqual(os.stat(f.fileno()).st_size, reduced_size)
997 finally:
998 f.close()
999
1000 @unittest.skipUnless(os.name == 'nt', 'requires Windows')
1001 def test_resize_succeeds_with_error_for_second_named_mapping(self):
1002 """If a more than one mapping exists of the same name, none of them can
1003 be resized: they'll raise an Exception and leave the original mapping intact
1004 """
1005 start_size = 2 * PAGESIZE
1006 reduced_size = PAGESIZE
1007 tagname = random_tagname()
1008 data_length = 8
1009 data = bytes(random.getrandbits(8) for _ in range(data_length))
1010
1011 m1 = mmap.mmap(-1, start_size, tagname=tagname)
1012 m2 = mmap.mmap(-1, start_size, tagname=tagname)
1013 m1[:data_length] = data
1014 self.assertEqual(m2[:data_length], data)
1015 with self.assertRaises(OSError):
1016 m1.resize(reduced_size)
1017 self.assertEqual(m1.size(), start_size)
1018 self.assertEqual(m1[:data_length], data)
1019 self.assertEqual(m2[:data_length], data)
1020
1021 def test_mmap_closed_by_int_scenarios(self):
1022 """
1023 gh-103987: Test that mmap objects raise ValueError
1024 for closed mmap files
1025 """
1026
1027 class MmapClosedByIntContext:
1028 def __init__(self, access) -> None:
1029 self.access = access
1030
1031 def __enter__(self):
1032 self.f = open(TESTFN, "w+b")
1033 self.f.write(random.randbytes(100))
1034 self.f.flush()
1035
1036 m = mmap.mmap(self.f.fileno(), 100, access=self.access)
1037
1038 class X:
1039 def __index__(self):
1040 m.close()
1041 return 10
1042
1043 return (m, X)
1044
1045 def __exit__(self, exc_type, exc_value, traceback):
1046 self.f.close()
1047
1048 read_access_modes = [
1049 mmap.ACCESS_READ,
1050 mmap.ACCESS_WRITE,
1051 mmap.ACCESS_COPY,
1052 mmap.ACCESS_DEFAULT,
1053 ]
1054
1055 write_access_modes = [
1056 mmap.ACCESS_WRITE,
1057 mmap.ACCESS_COPY,
1058 mmap.ACCESS_DEFAULT,
1059 ]
1060
1061 for access in read_access_modes:
1062 with MmapClosedByIntContext(access) as (m, X):
1063 with self.assertRaisesRegex(ValueError, "mmap closed or invalid"):
1064 m[X()]
1065
1066 with MmapClosedByIntContext(access) as (m, X):
1067 with self.assertRaisesRegex(ValueError, "mmap closed or invalid"):
1068 m[X() : 20]
1069
1070 with MmapClosedByIntContext(access) as (m, X):
1071 with self.assertRaisesRegex(ValueError, "mmap closed or invalid"):
1072 m[X() : 20 : 2]
1073
1074 with MmapClosedByIntContext(access) as (m, X):
1075 with self.assertRaisesRegex(ValueError, "mmap closed or invalid"):
1076 m[20 : X() : -2]
1077
1078 with MmapClosedByIntContext(access) as (m, X):
1079 with self.assertRaisesRegex(ValueError, "mmap closed or invalid"):
1080 m.read(X())
1081
1082 with MmapClosedByIntContext(access) as (m, X):
1083 with self.assertRaisesRegex(ValueError, "mmap closed or invalid"):
1084 m.find(b"1", 1, X())
1085
1086 for access in write_access_modes:
1087 with MmapClosedByIntContext(access) as (m, X):
1088 with self.assertRaisesRegex(ValueError, "mmap closed or invalid"):
1089 m[X() : 20] = b"1" * 10
1090
1091 with MmapClosedByIntContext(access) as (m, X):
1092 with self.assertRaisesRegex(ValueError, "mmap closed or invalid"):
1093 m[X() : 20 : 2] = b"1" * 5
1094
1095 with MmapClosedByIntContext(access) as (m, X):
1096 with self.assertRaisesRegex(ValueError, "mmap closed or invalid"):
1097 m[20 : X() : -2] = b"1" * 5
1098
1099 with MmapClosedByIntContext(access) as (m, X):
1100 with self.assertRaisesRegex(ValueError, "mmap closed or invalid"):
1101 m.move(1, 2, X())
1102
1103 with MmapClosedByIntContext(access) as (m, X):
1104 with self.assertRaisesRegex(ValueError, "mmap closed or invalid"):
1105 m.write_byte(X())
1106
1107 @unittest.skipUnless(os.name == 'nt', 'requires Windows')
1108 @unittest.skipUnless(hasattr(mmap.mmap, '_protect'), 'test needs debug build')
1109 def test_access_violations(self):
1110 from test.support.os_helper import TESTFN
1111
1112 code = textwrap.dedent("""
1113 import faulthandler
1114 import mmap
1115 import os
1116 import sys
1117 from contextlib import suppress
1118
1119 # Prevent logging access violations to stderr.
1120 faulthandler.disable()
1121
1122 PAGESIZE = mmap.PAGESIZE
1123 PAGE_NOACCESS = 0x01
1124
1125 with open(sys.argv[1], 'bw+') as f:
1126 f.write(b'A'* PAGESIZE)
1127 f.flush()
1128
1129 m = mmap.mmap(f.fileno(), PAGESIZE)
1130 m._protect(PAGE_NOACCESS, 0, PAGESIZE)
1131 with suppress(OSError):
1132 m.read(PAGESIZE)
1133 assert False, 'mmap.read() did not raise'
1134 with suppress(OSError):
1135 m.read_byte()
1136 assert False, 'mmap.read_byte() did not raise'
1137 with suppress(OSError):
1138 m.readline()
1139 assert False, 'mmap.readline() did not raise'
1140 with suppress(OSError):
1141 m.write(b'A'* PAGESIZE)
1142 assert False, 'mmap.write() did not raise'
1143 with suppress(OSError):
1144 m.write_byte(0)
1145 assert False, 'mmap.write_byte() did not raise'
1146 with suppress(OSError):
1147 m[0] # test mmap_subscript
1148 assert False, 'mmap.__getitem__() did not raise'
1149 with suppress(OSError):
1150 m[0:10] # test mmap_subscript
1151 assert False, 'mmap.__getitem__() did not raise'
1152 with suppress(OSError):
1153 m[0:10:2] # test mmap_subscript
1154 assert False, 'mmap.__getitem__() did not raise'
1155 with suppress(OSError):
1156 m[0] = 1
1157 assert False, 'mmap.__setitem__() did not raise'
1158 with suppress(OSError):
1159 m[0:10] = b'A'* 10
1160 assert False, 'mmap.__setitem__() did not raise'
1161 with suppress(OSError):
1162 m[0:10:2] = b'A'* 5
1163 assert False, 'mmap.__setitem__() did not raise'
1164 with suppress(OSError):
1165 m.move(0, 10, 1)
1166 assert False, 'mmap.move() did not raise'
1167 with suppress(OSError):
1168 list(m) # test mmap_item
1169 assert False, 'mmap.__getitem__() did not raise'
1170 with suppress(OSError):
1171 m.find(b'A')
1172 assert False, 'mmap.find() did not raise'
1173 with suppress(OSError):
1174 m.rfind(b'A')
1175 assert False, 'mmap.rfind() did not raise'
1176 """)
1177 rt, stdout, stderr = assert_python_ok("-c", code, TESTFN)
1178 self.assertEqual(stdout.strip(), b'')
1179 self.assertEqual(stderr.strip(), b'')
1180
1181
1182 class LargeMmapTests(unittest.TestCase):
1183
1184 def setUp(self):
1185 unlink(TESTFN)
1186
1187 def tearDown(self):
1188 unlink(TESTFN)
1189
1190 def _make_test_file(self, num_zeroes, tail):
1191 if sys.platform[:3] == 'win' or is_apple:
1192 requires('largefile',
1193 'test requires %s bytes and a long time to run' % str(0x180000000))
1194 f = open(TESTFN, 'w+b')
1195 try:
1196 f.seek(num_zeroes)
1197 f.write(tail)
1198 f.flush()
1199 except (OSError, OverflowError, ValueError):
1200 try:
1201 f.close()
1202 except (OSError, OverflowError):
1203 pass
1204 raise unittest.SkipTest("filesystem does not have largefile support")
1205 return f
1206
1207 def test_large_offset(self):
1208 with self._make_test_file(0x14FFFFFFF, b" ") as f:
1209 with mmap.mmap(f.fileno(), 0, offset=0x140000000, access=mmap.ACCESS_READ) as m:
1210 self.assertEqual(m[0xFFFFFFF], 32)
1211
1212 def test_large_filesize(self):
1213 with self._make_test_file(0x17FFFFFFF, b" ") as f:
1214 if sys.maxsize < 0x180000000:
1215 # On 32 bit platforms the file is larger than sys.maxsize so
1216 # mapping the whole file should fail -- Issue #16743
1217 with self.assertRaises(OverflowError):
1218 mmap.mmap(f.fileno(), 0x180000000, access=mmap.ACCESS_READ)
1219 with self.assertRaises(ValueError):
1220 mmap.mmap(f.fileno(), 0, access=mmap.ACCESS_READ)
1221 with mmap.mmap(f.fileno(), 0x10000, access=mmap.ACCESS_READ) as m:
1222 self.assertEqual(m.size(), 0x180000000)
1223
1224 # Issue 11277: mmap() with large (~4 GiB) sparse files crashes on OS X.
1225
1226 def _test_around_boundary(self, boundary):
1227 tail = b' DEARdear '
1228 start = boundary - len(tail) // 2
1229 end = start + len(tail)
1230 with self._make_test_file(start, tail) as f:
1231 with mmap.mmap(f.fileno(), 0, access=mmap.ACCESS_READ) as m:
1232 self.assertEqual(m[start:end], tail)
1233
1234 @unittest.skipUnless(sys.maxsize > _4G, "test cannot run on 32-bit systems")
1235 def test_around_2GB(self):
1236 self._test_around_boundary(_2G)
1237
1238 @unittest.skipUnless(sys.maxsize > _4G, "test cannot run on 32-bit systems")
1239 def test_around_4GB(self):
1240 self._test_around_boundary(_4G)
1241
1242
1243 if __name__ == '__main__':
1244 unittest.main()