Debian Codegraph @

Lib/test/test_zipimport.py

raw
1 import sys
2 import os
3 import marshal
4 import glob
5 import importlib
6 import importlib.util
7 import re
8 import struct
9 import time
10 import unittest
11 import unittest.mock
12 import warnings
13
14 from test import support
15 from test.support import import_helper
16 from test.support import os_helper
17
18 from zipfile import ZipFile, ZipInfo, ZIP_STORED, ZIP_DEFLATED
19
20 import zipimport
21 import linecache
22 import doctest
23 import inspect
24 import io
25 from traceback import extract_tb, extract_stack, print_tb
26 try:
27 import zlib
28 except ImportError:
29 zlib = None
30
31 test_src = """\
32 def get_name():
33 return __name__
34 def get_file():
35 return __file__
36 """
37 test_co = compile(test_src, "<???>", "exec")
38 raise_src = 'def do_raise(): raise TypeError\n'
39
40 def make_pyc(co, mtime, size):
41 data = marshal.dumps(co)
42 pyc = (importlib.util.MAGIC_NUMBER +
43 struct.pack("<iLL", 0,
44 int(mtime) & 0xFFFF_FFFF, size & 0xFFFF_FFFF) + data)
45 return pyc
46
47 def module_path_to_dotted_name(path):
48 return path.replace(os.sep, '.')
49
50 NOW = time.time()
51 test_pyc = make_pyc(test_co, NOW, len(test_src))
52
53
54 TESTMOD = "ziptestmodule"
55 TESTMOD2 = "ziptestmodule2"
56 TESTMOD3 = "ziptestmodule3"
57 TESTPACK = "ziptestpackage"
58 TESTPACK2 = "ziptestpackage2"
59 TESTPACK3 = "ziptestpackage3"
60 TEMP_DIR = os.path.abspath("junk95142")
61 TEMP_ZIP = os.path.abspath("junk95142.zip")
62 TEST_DATA_DIR = os.path.join(os.path.dirname(__file__), "zipimport_data")
63
64 pyc_file = importlib.util.cache_from_source(TESTMOD + '.py')
65 pyc_ext = '.pyc'
66
67
68 class ImportHooksBaseTestCase(unittest.TestCase):
69
70 def setUp(self):
71 self.path = sys.path[:]
72 self.meta_path = sys.meta_path[:]
73 self.path_hooks = sys.path_hooks[:]
74 sys.path_importer_cache.clear()
75 self.modules_before = import_helper.modules_setup()
76
77 def tearDown(self):
78 sys.path[:] = self.path
79 sys.meta_path[:] = self.meta_path
80 sys.path_hooks[:] = self.path_hooks
81 sys.path_importer_cache.clear()
82 import_helper.modules_cleanup(*self.modules_before)
83
84
85 class UncompressedZipImportTestCase(ImportHooksBaseTestCase):
86
87 compression = ZIP_STORED
88
89 def setUp(self):
90 # We're reusing the zip archive path, so we must clear the
91 # cached directory info and linecache.
92 linecache.clearcache()
93 zipimport._zip_directory_cache.clear()
94 ImportHooksBaseTestCase.setUp(self)
95
96 def makeTree(self, files, dirName=TEMP_DIR):
97 # Create a filesystem based set of modules/packages
98 # defined by files under the directory dirName.
99 self.addCleanup(os_helper.rmtree, dirName)
100
101 for name, data in files.items():
102 if isinstance(data, tuple):
103 mtime, data = data
104 path = os.path.join(dirName, *name.split('/'))
105 if path[-1] == os.sep:
106 if not os.path.isdir(path):
107 os.makedirs(path)
108 else:
109 dname = os.path.dirname(path)
110 if not os.path.isdir(dname):
111 os.makedirs(dname)
112 with open(path, 'wb') as fp:
113 fp.write(data)
114
115 def makeZip(self, files, zipName=TEMP_ZIP, *,
116 comment=None, file_comment=None, stuff=None, prefix='', **kw):
117 # Create a zip archive based set of modules/packages
118 # defined by files in the zip file zipName.
119 # If stuff is not None, it is prepended to the archive.
120 self.addCleanup(os_helper.unlink, zipName)
121
122 with ZipFile(zipName, "w", compression=self.compression) as z:
123 self.writeZip(z, files, file_comment=file_comment, prefix=prefix)
124 if comment is not None:
125 z.comment = comment
126
127 if stuff is not None:
128 # Prepend 'stuff' to the start of the zipfile
129 with open(zipName, "rb") as f:
130 data = f.read()
131 with open(zipName, "wb") as f:
132 f.write(stuff)
133 f.write(data)
134
135 def writeZip(self, z, files, *, file_comment=None, prefix=''):
136 for name, data in files.items():
137 if isinstance(data, tuple):
138 mtime, data = data
139 else:
140 mtime = NOW
141 name = name.replace(os.sep, '/')
142 zinfo = ZipInfo(prefix + name, time.localtime(mtime))
143 zinfo.compress_type = self.compression
144 if file_comment is not None:
145 zinfo.comment = file_comment
146 if data is None:
147 zinfo.CRC = 0
148 z.mkdir(zinfo)
149 else:
150 assert name[-1] != '/'
151 z.writestr(zinfo, data)
152
153 def getZip64Files(self):
154 # This is the simplest way to make zipfile generate the zip64 EOCD block
155 return {f"f{n}.py": test_src for n in range(65537)}
156
157 def doTest(self, expected_ext, files, *modules, **kw):
158 if 'prefix' not in kw:
159 kw['prefix'] = 'pre/fix/'
160 self.makeZip(files, **kw)
161 self.doTestWithPreBuiltZip(expected_ext, *modules, **kw)
162
163 def doTestWithPreBuiltZip(self, expected_ext, *modules,
164 call=None, prefix='', **kw):
165 zip_path = os.path.join(TEMP_ZIP, *prefix.split('/')[:-1])
166 sys.path.insert(0, zip_path)
167
168 mod = importlib.import_module(".".join(modules))
169
170 if call is not None:
171 call(mod)
172
173 if expected_ext:
174 file = mod.get_file()
175 self.assertEqual(file, os.path.join(zip_path,
176 *modules) + expected_ext)
177
178 def testAFakeZlib(self):
179 #
180 # This could cause a stack overflow before: importing zlib.py
181 # from a compressed archive would cause zlib to be imported
182 # which would find zlib.py in the archive, which would... etc.
183 #
184 # This test *must* be executed first: it must be the first one
185 # to trigger zipimport to import zlib (zipimport caches the
186 # zlib.decompress function object, after which the problem being
187 # tested here wouldn't be a problem anymore...
188 # (Hence the 'A' in the test method name: to make it the first
189 # item in a list sorted by name, like
190 # unittest.TestLoader.getTestCaseNames() does.)
191 #
192 # This test fails on platforms on which the zlib module is
193 # statically linked, but the problem it tests for can't
194 # occur in that case (builtin modules are always found first),
195 # so we'll simply skip it then. Bug #765456.
196 #
197 if "zlib" in sys.builtin_module_names:
198 self.skipTest('zlib is a builtin module')
199 if "zlib" in sys.modules:
200 del sys.modules["zlib"]
201 files = {"zlib.py": test_src}
202 try:
203 self.doTest(".py", files, "zlib")
204 except ImportError:
205 if self.compression != ZIP_DEFLATED:
206 self.fail("expected test to not raise ImportError")
207 else:
208 if self.compression != ZIP_STORED:
209 self.fail("expected test to raise ImportError")
210
211 def testPy(self):
212 files = {TESTMOD + ".py": test_src}
213 self.doTest(".py", files, TESTMOD)
214
215 def testPyc(self):
216 files = {TESTMOD + pyc_ext: test_pyc}
217 self.doTest(pyc_ext, files, TESTMOD)
218
219 def testBoth(self):
220 files = {TESTMOD + ".py": test_src,
221 TESTMOD + pyc_ext: test_pyc}
222 self.doTest(pyc_ext, files, TESTMOD)
223
224 def testUncheckedHashBasedPyc(self):
225 source = b"state = 'old'"
226 source_hash = importlib.util.source_hash(source)
227 bytecode = importlib._bootstrap_external._code_to_hash_pyc(
228 compile(source, "???", "exec"),
229 source_hash,
230 False, # unchecked
231 )
232 files = {TESTMOD + ".py": (NOW, "state = 'new'"),
233 TESTMOD + ".pyc": (NOW - 20, bytecode)}
234 def check(mod):
235 self.assertEqual(mod.state, 'old')
236 self.doTest(None, files, TESTMOD, call=check)
237
238 @unittest.mock.patch('_imp.check_hash_based_pycs', 'always')
239 def test_checked_hash_based_change_pyc(self):
240 source = b"state = 'old'"
241 source_hash = importlib.util.source_hash(source)
242 bytecode = importlib._bootstrap_external._code_to_hash_pyc(
243 compile(source, "???", "exec"),
244 source_hash,
245 False,
246 )
247 files = {TESTMOD + ".py": (NOW, "state = 'new'"),
248 TESTMOD + ".pyc": (NOW - 20, bytecode)}
249 def check(mod):
250 self.assertEqual(mod.state, 'new')
251 self.doTest(None, files, TESTMOD, call=check)
252
253 def testEmptyPy(self):
254 files = {TESTMOD + ".py": ""}
255 self.doTest(None, files, TESTMOD)
256
257 def testBadMagic(self):
258 # make pyc magic word invalid, forcing loading from .py
259 badmagic_pyc = bytearray(test_pyc)
260 badmagic_pyc[0] ^= 0x04 # flip an arbitrary bit
261 files = {TESTMOD + ".py": test_src,
262 TESTMOD + pyc_ext: badmagic_pyc}
263 self.doTest(".py", files, TESTMOD)
264
265 def testBadMagic2(self):
266 # make pyc magic word invalid, causing an ImportError
267 badmagic_pyc = bytearray(test_pyc)
268 badmagic_pyc[0] ^= 0x04 # flip an arbitrary bit
269 files = {TESTMOD + pyc_ext: badmagic_pyc}
270 try:
271 self.doTest(".py", files, TESTMOD)
272 self.fail("This should not be reached")
273 except zipimport.ZipImportError as exc:
274 self.assertIsInstance(exc.__cause__, ImportError)
275 self.assertIn("magic number", exc.__cause__.msg)
276
277 def testBadMTime(self):
278 badtime_pyc = bytearray(test_pyc)
279 # flip the second bit -- not the first as that one isn't stored in the
280 # .py's mtime in the zip archive.
281 badtime_pyc[11] ^= 0x02
282 files = {TESTMOD + ".py": test_src,
283 TESTMOD + pyc_ext: badtime_pyc}
284 self.doTest(".py", files, TESTMOD)
285
286 def test2038MTime(self):
287 # Make sure we can handle mtimes larger than what a 32-bit signed number
288 # can hold.
289 twenty_thirty_eight_pyc = make_pyc(test_co, 2**32 - 1, len(test_src))
290 files = {TESTMOD + ".py": test_src,
291 TESTMOD + pyc_ext: twenty_thirty_eight_pyc}
292 self.doTest(".py", files, TESTMOD)
293
294 def testPackage(self):
295 packdir = TESTPACK + os.sep
296 files = {packdir + "__init__" + pyc_ext: test_pyc,
297 packdir + TESTMOD + pyc_ext: test_pyc}
298 self.doTest(pyc_ext, files, TESTPACK, TESTMOD)
299
300 def testSubPackage(self):
301 # Test that subpackages function when loaded from zip
302 # archives.
303 packdir = TESTPACK + os.sep
304 packdir2 = packdir + TESTPACK2 + os.sep
305 files = {packdir + "__init__" + pyc_ext: test_pyc,
306 packdir2 + "__init__" + pyc_ext: test_pyc,
307 packdir2 + TESTMOD + pyc_ext: test_pyc}
308 self.doTest(pyc_ext, files, TESTPACK, TESTPACK2, TESTMOD)
309
310 def testSubNamespacePackage(self):
311 # Test that implicit namespace subpackages function
312 # when loaded from zip archives.
313 packdir = TESTPACK + os.sep
314 packdir2 = packdir + TESTPACK2 + os.sep
315 # The first two files are just directory entries (so have no data).
316 files = {packdir: None,
317 packdir2: None,
318 packdir2 + TESTMOD + pyc_ext: test_pyc}
319 self.doTest(pyc_ext, files, TESTPACK, TESTPACK2, TESTMOD)
320
321 def testPackageExplicitDirectories(self):
322 # Test explicit namespace packages with explicit directory entries.
323 self.addCleanup(os_helper.unlink, TEMP_ZIP)
324 with ZipFile(TEMP_ZIP, 'w', compression=self.compression) as z:
325 z.mkdir('a')
326 z.writestr('a/__init__.py', test_src)
327 z.mkdir('a/b')
328 z.writestr('a/b/__init__.py', test_src)
329 z.mkdir('a/b/c')
330 z.writestr('a/b/c/__init__.py', test_src)
331 z.writestr('a/b/c/d.py', test_src)
332 self._testPackage(initfile='__init__.py')
333
334 def testPackageImplicitDirectories(self):
335 # Test explicit namespace packages without explicit directory entries.
336 self.addCleanup(os_helper.unlink, TEMP_ZIP)
337 with ZipFile(TEMP_ZIP, 'w', compression=self.compression) as z:
338 z.writestr('a/__init__.py', test_src)
339 z.writestr('a/b/__init__.py', test_src)
340 z.writestr('a/b/c/__init__.py', test_src)
341 z.writestr('a/b/c/d.py', test_src)
342 self._testPackage(initfile='__init__.py')
343
344 def testNamespacePackageExplicitDirectories(self):
345 # Test implicit namespace packages with explicit directory entries.
346 self.addCleanup(os_helper.unlink, TEMP_ZIP)
347 with ZipFile(TEMP_ZIP, 'w', compression=self.compression) as z:
348 z.mkdir('a')
349 z.mkdir('a/b')
350 z.mkdir('a/b/c')
351 z.writestr('a/b/c/d.py', test_src)
352 self._testPackage(initfile=None)
353
354 def testNamespacePackageImplicitDirectories(self):
355 # Test implicit namespace packages without explicit directory entries.
356 self.addCleanup(os_helper.unlink, TEMP_ZIP)
357 with ZipFile(TEMP_ZIP, 'w', compression=self.compression) as z:
358 z.writestr('a/b/c/d.py', test_src)
359 self._testPackage(initfile=None)
360
361 def _testPackage(self, initfile):
362 zi = zipimport.zipimporter(os.path.join(TEMP_ZIP, 'a'))
363 if initfile is None:
364 # XXX Should it work?
365 self.assertRaises(zipimport.ZipImportError, zi.is_package, 'b')
366 self.assertRaises(zipimport.ZipImportError, zi.get_source, 'b')
367 self.assertRaises(zipimport.ZipImportError, zi.get_code, 'b')
368 else:
369 self.assertTrue(zi.is_package('b'))
370 self.assertEqual(zi.get_source('b'), test_src)
371 self.assertEqual(zi.get_code('b').co_filename,
372 os.path.join(TEMP_ZIP, 'a', 'b', initfile))
373
374 sys.path.insert(0, TEMP_ZIP)
375 self.assertNotIn('a', sys.modules)
376
377 mod = importlib.import_module(f'a.b')
378 self.assertIn('a', sys.modules)
379 self.assertIs(sys.modules['a.b'], mod)
380 if initfile is None:
381 self.assertIsNone(mod.__file__)
382 else:
383 self.assertEqual(mod.__file__,
384 os.path.join(TEMP_ZIP, 'a', 'b', initfile))
385 self.assertEqual(len(mod.__path__), 1, mod.__path__)
386 self.assertEqual(mod.__path__[0], os.path.join(TEMP_ZIP, 'a', 'b'))
387
388 mod2 = importlib.import_module(f'a.b.c.d')
389 self.assertIn('a.b.c', sys.modules)
390 self.assertIn('a.b.c.d', sys.modules)
391 self.assertIs(sys.modules['a.b.c.d'], mod2)
392 self.assertIs(mod.c.d, mod2)
393 self.assertEqual(mod2.__file__,
394 os.path.join(TEMP_ZIP, 'a', 'b', 'c', 'd.py'))
395
396 def testMixedNamespacePackage(self):
397 # Test implicit namespace packages spread between a
398 # real filesystem and a zip archive.
399 packdir = TESTPACK + os.sep
400 packdir2 = packdir + TESTPACK2 + os.sep
401 packdir3 = packdir2 + TESTPACK3 + os.sep
402 files1 = {packdir: None,
403 packdir + TESTMOD + pyc_ext: test_pyc,
404 packdir2: None,
405 packdir3: None,
406 packdir3 + TESTMOD + pyc_ext: test_pyc,
407 packdir2 + TESTMOD3 + pyc_ext: test_pyc,
408 packdir2 + TESTMOD + pyc_ext: test_pyc}
409 files2 = {packdir: None,
410 packdir + TESTMOD2 + pyc_ext: test_pyc,
411 packdir2: None,
412 packdir2 + TESTMOD2 + pyc_ext: test_pyc,
413 packdir2 + TESTMOD + pyc_ext: test_pyc}
414
415 zip1 = os.path.abspath("path1.zip")
416 self.makeZip(files1, zip1)
417
418 zip2 = TEMP_DIR
419 self.makeTree(files2, zip2)
420
421 # zip2 should override zip1.
422 sys.path.insert(0, zip1)
423 sys.path.insert(0, zip2)
424
425 mod = importlib.import_module(TESTPACK)
426
427 # if TESTPACK is functioning as a namespace pkg then
428 # there should be two entries in the __path__.
429 # First should be path2 and second path1.
430 self.assertEqual(2, len(mod.__path__))
431 p1, p2 = mod.__path__
432 self.assertEqual(os.path.basename(TEMP_DIR), p1.split(os.sep)[-2])
433 self.assertEqual("path1.zip", p2.split(os.sep)[-2])
434
435 # packdir3 should import as a namespace package.
436 # Its __path__ is an iterable of 1 element from zip1.
437 mod = importlib.import_module(packdir3.replace(os.sep, '.')[:-1])
438 self.assertEqual(1, len(mod.__path__))
439 mpath = list(mod.__path__)[0].split('path1.zip' + os.sep)[1]
440 self.assertEqual(packdir3[:-1], mpath)
441
442 # TESTPACK/TESTMOD only exists in path1.
443 mod = importlib.import_module('.'.join((TESTPACK, TESTMOD)))
444 self.assertEqual("path1.zip", mod.__file__.split(os.sep)[-3])
445
446 # And TESTPACK/(TESTMOD2) only exists in path2.
447 mod = importlib.import_module('.'.join((TESTPACK, TESTMOD2)))
448 self.assertEqual(os.path.basename(TEMP_DIR),
449 mod.__file__.split(os.sep)[-3])
450
451 # One level deeper...
452 subpkg = '.'.join((TESTPACK, TESTPACK2))
453 mod = importlib.import_module(subpkg)
454 self.assertEqual(2, len(mod.__path__))
455 p1, p2 = mod.__path__
456 self.assertEqual(os.path.basename(TEMP_DIR), p1.split(os.sep)[-3])
457 self.assertEqual("path1.zip", p2.split(os.sep)[-3])
458
459 # subpkg.TESTMOD exists in both zips should load from zip2.
460 mod = importlib.import_module('.'.join((subpkg, TESTMOD)))
461 self.assertEqual(os.path.basename(TEMP_DIR),
462 mod.__file__.split(os.sep)[-4])
463
464 # subpkg.TESTMOD2 only exists in zip2.
465 mod = importlib.import_module('.'.join((subpkg, TESTMOD2)))
466 self.assertEqual(os.path.basename(TEMP_DIR),
467 mod.__file__.split(os.sep)[-4])
468
469 # Finally subpkg.TESTMOD3 only exists in zip1.
470 mod = importlib.import_module('.'.join((subpkg, TESTMOD3)))
471 self.assertEqual('path1.zip', mod.__file__.split(os.sep)[-4])
472
473 def testNamespacePackage(self):
474 # Test implicit namespace packages spread between multiple zip
475 # archives.
476 packdir = TESTPACK + os.sep
477 packdir2 = packdir + TESTPACK2 + os.sep
478 packdir3 = packdir2 + TESTPACK3 + os.sep
479 files1 = {packdir: None,
480 packdir + TESTMOD + pyc_ext: test_pyc,
481 packdir2: None,
482 packdir3: None,
483 packdir3 + TESTMOD + pyc_ext: test_pyc,
484 packdir2 + TESTMOD3 + pyc_ext: test_pyc,
485 packdir2 + TESTMOD + pyc_ext: test_pyc}
486 zip1 = os.path.abspath("path1.zip")
487 self.makeZip(files1, zip1)
488
489 files2 = {packdir: None,
490 packdir + TESTMOD2 + pyc_ext: test_pyc,
491 packdir2: None,
492 packdir2 + TESTMOD2 + pyc_ext: test_pyc,
493 packdir2 + TESTMOD + pyc_ext: test_pyc}
494 zip2 = os.path.abspath("path2.zip")
495 self.makeZip(files2, zip2)
496
497 # zip2 should override zip1.
498 sys.path.insert(0, zip1)
499 sys.path.insert(0, zip2)
500
501 mod = importlib.import_module(TESTPACK)
502
503 # if TESTPACK is functioning as a namespace pkg then
504 # there should be two entries in the __path__.
505 # First should be path2 and second path1.
506 self.assertEqual(2, len(mod.__path__))
507 p1, p2 = mod.__path__
508 self.assertEqual("path2.zip", p1.split(os.sep)[-2])
509 self.assertEqual("path1.zip", p2.split(os.sep)[-2])
510
511 # packdir3 should import as a namespace package.
512 # Tts __path__ is an iterable of 1 element from zip1.
513 mod = importlib.import_module(packdir3.replace(os.sep, '.')[:-1])
514 self.assertEqual(1, len(mod.__path__))
515 mpath = list(mod.__path__)[0].split('path1.zip' + os.sep)[1]
516 self.assertEqual(packdir3[:-1], mpath)
517
518 # TESTPACK/TESTMOD only exists in path1.
519 mod = importlib.import_module('.'.join((TESTPACK, TESTMOD)))
520 self.assertEqual("path1.zip", mod.__file__.split(os.sep)[-3])
521
522 # And TESTPACK/(TESTMOD2) only exists in path2.
523 mod = importlib.import_module('.'.join((TESTPACK, TESTMOD2)))
524 self.assertEqual("path2.zip", mod.__file__.split(os.sep)[-3])
525
526 # One level deeper...
527 subpkg = '.'.join((TESTPACK, TESTPACK2))
528 mod = importlib.import_module(subpkg)
529 self.assertEqual(2, len(mod.__path__))
530 p1, p2 = mod.__path__
531 self.assertEqual("path2.zip", p1.split(os.sep)[-3])
532 self.assertEqual("path1.zip", p2.split(os.sep)[-3])
533
534 # subpkg.TESTMOD exists in both zips should load from zip2.
535 mod = importlib.import_module('.'.join((subpkg, TESTMOD)))
536 self.assertEqual('path2.zip', mod.__file__.split(os.sep)[-4])
537
538 # subpkg.TESTMOD2 only exists in zip2.
539 mod = importlib.import_module('.'.join((subpkg, TESTMOD2)))
540 self.assertEqual('path2.zip', mod.__file__.split(os.sep)[-4])
541
542 # Finally subpkg.TESTMOD3 only exists in zip1.
543 mod = importlib.import_module('.'.join((subpkg, TESTMOD3)))
544 self.assertEqual('path1.zip', mod.__file__.split(os.sep)[-4])
545
546 def testZipImporterMethods(self):
547 packdir = TESTPACK + os.sep
548 packdir2 = packdir + TESTPACK2 + os.sep
549 files = {packdir + "__init__" + pyc_ext: test_pyc,
550 packdir2 + "__init__" + pyc_ext: test_pyc,
551 packdir2 + TESTMOD + pyc_ext: test_pyc,
552 "spam" + pyc_ext: test_pyc}
553 self.makeZip(files, file_comment=b"spam")
554
555 zi = zipimport.zipimporter(TEMP_ZIP)
556 self.assertEqual(zi.archive, TEMP_ZIP)
557 self.assertTrue(zi.is_package(TESTPACK))
558
559 # PEP 302
560 with warnings.catch_warnings():
561 warnings.simplefilter("ignore", DeprecationWarning)
562
563 mod = zi.load_module(TESTPACK)
564 self.assertEqual(zi.get_filename(TESTPACK), mod.__file__)
565
566 # PEP 451
567 spec = zi.find_spec('spam')
568 self.assertIsNotNone(spec)
569 self.assertIsInstance(spec.loader, zipimport.zipimporter)
570 self.assertFalse(spec.loader.is_package('spam'))
571 exec_mod = importlib.util.module_from_spec(spec)
572 spec.loader.exec_module(exec_mod)
573 self.assertEqual(spec.loader.get_filename('spam'), exec_mod.__file__)
574
575 spec = zi.find_spec(TESTPACK)
576 mod = importlib.util.module_from_spec(spec)
577 spec.loader.exec_module(mod)
578 self.assertEqual(zi.get_filename(TESTPACK), mod.__file__)
579
580 existing_pack_path = importlib.import_module(TESTPACK).__path__[0]
581 expected_path_path = os.path.join(TEMP_ZIP, TESTPACK)
582 self.assertEqual(existing_pack_path, expected_path_path)
583
584 self.assertFalse(zi.is_package(packdir + '__init__'))
585 self.assertTrue(zi.is_package(packdir + TESTPACK2))
586 self.assertFalse(zi.is_package(packdir2 + TESTMOD))
587
588 mod_path = packdir2 + TESTMOD
589 mod_name = module_path_to_dotted_name(mod_path)
590 mod = importlib.import_module(mod_name)
591 self.assertTrue(mod_name in sys.modules)
592 self.assertIsNone(zi.get_source(TESTPACK))
593 self.assertIsNone(zi.get_source(mod_path))
594 self.assertEqual(zi.get_filename(mod_path), mod.__file__)
595 # To pass in the module name instead of the path, we must use the
596 # right importer
597 loader = mod.__spec__.loader
598 self.assertIsNone(loader.get_source(mod_name))
599 self.assertEqual(loader.get_filename(mod_name), mod.__file__)
600
601 # test prefix and archivepath members
602 zi2 = zipimport.zipimporter(TEMP_ZIP + os.sep + TESTPACK)
603 self.assertEqual(zi2.archive, TEMP_ZIP)
604 self.assertEqual(zi2.prefix, TESTPACK + os.sep)
605
606 def testInvalidateCaches(self):
607 packdir = TESTPACK + os.sep
608 packdir2 = packdir + TESTPACK2 + os.sep
609 files = {packdir + "__init__" + pyc_ext: test_pyc,
610 packdir2 + "__init__" + pyc_ext: test_pyc,
611 packdir2 + TESTMOD + pyc_ext: test_pyc,
612 "spam" + pyc_ext: test_pyc}
613 extra_files = [packdir, packdir2]
614 self.makeZip(files, file_comment=b"spam")
615
616 zi = zipimport.zipimporter(TEMP_ZIP)
617 self.assertEqual(sorted(zi._get_files()), sorted([*files, *extra_files]))
618 # Check that the file information remains accurate after reloading
619 zi.invalidate_caches()
620 self.assertEqual(sorted(zi._get_files()), sorted([*files, *extra_files]))
621 # Add a new file to the ZIP archive
622 newfile = {"spam2" + pyc_ext: test_pyc}
623 files.update(newfile)
624 with ZipFile(TEMP_ZIP, "a", compression=self.compression) as z:
625 self.writeZip(z, newfile, file_comment=b"spam")
626 # Check that we can detect the new file after invalidating the cache
627 zi.invalidate_caches()
628 self.assertEqual(sorted(zi._get_files()), sorted([*files, *extra_files]))
629 spec = zi.find_spec('spam2')
630 self.assertIsNotNone(spec)
631 self.assertIsInstance(spec.loader, zipimport.zipimporter)
632 # Check that the cached data is removed if the file is deleted
633 os.remove(TEMP_ZIP)
634 zi.invalidate_caches()
635 self.assertFalse(zi._get_files())
636 self.assertIsNone(zipimport._zip_directory_cache.get(zi.archive))
637 self.assertIsNone(zi.find_spec("name_does_not_matter"))
638
639 def testInvalidateCachesWithMultipleZipimports(self):
640 packdir = TESTPACK + os.sep
641 packdir2 = packdir + TESTPACK2 + os.sep
642 files = {packdir + "__init__" + pyc_ext: test_pyc,
643 packdir2 + "__init__" + pyc_ext: test_pyc,
644 packdir2 + TESTMOD + pyc_ext: test_pyc,
645 "spam" + pyc_ext: test_pyc}
646 extra_files = [packdir, packdir2]
647 self.makeZip(files, file_comment=b"spam")
648
649 zi = zipimport.zipimporter(TEMP_ZIP)
650 self.assertEqual(sorted(zi._get_files()), sorted([*files, *extra_files]))
651 # Zipimporter for the same path.
652 zi2 = zipimport.zipimporter(TEMP_ZIP)
653 self.assertEqual(sorted(zi2._get_files()), sorted([*files, *extra_files]))
654 # Add a new file to the ZIP archive to make the cache wrong.
655 newfile = {"spam2" + pyc_ext: test_pyc}
656 files.update(newfile)
657 with ZipFile(TEMP_ZIP, "a", compression=self.compression) as z:
658 self.writeZip(z, newfile, file_comment=b"spam")
659 # Invalidate the cache of the first zipimporter.
660 zi.invalidate_caches()
661 # Check that the second zipimporter detects the new file and isn't using a stale cache.
662 self.assertEqual(sorted(zi2._get_files()), sorted([*files, *extra_files]))
663 spec = zi2.find_spec('spam2')
664 self.assertIsNotNone(spec)
665 self.assertIsInstance(spec.loader, zipimport.zipimporter)
666
667 def testZipImporterMethodsInSubDirectory(self):
668 packdir = TESTPACK + os.sep
669 packdir2 = packdir + TESTPACK2 + os.sep
670 files = {packdir2 + "__init__" + pyc_ext: test_pyc,
671 packdir2 + TESTMOD + pyc_ext: test_pyc}
672 self.makeZip(files, file_comment=b"eggs")
673
674 zi = zipimport.zipimporter(TEMP_ZIP + os.sep + packdir)
675 self.assertEqual(zi.archive, TEMP_ZIP)
676 self.assertEqual(zi.prefix, packdir)
677 self.assertTrue(zi.is_package(TESTPACK2))
678 # PEP 302
679 with warnings.catch_warnings():
680 warnings.simplefilter("ignore", DeprecationWarning)
681 mod = zi.load_module(TESTPACK2)
682 self.assertEqual(zi.get_filename(TESTPACK2), mod.__file__)
683 # PEP 451
684 spec = zi.find_spec(TESTPACK2)
685 mod = importlib.util.module_from_spec(spec)
686 spec.loader.exec_module(mod)
687 self.assertEqual(spec.loader.get_filename(TESTPACK2), mod.__file__)
688
689 self.assertFalse(zi.is_package(TESTPACK2 + os.sep + '__init__'))
690 self.assertFalse(zi.is_package(TESTPACK2 + os.sep + TESTMOD))
691
692 pkg_path = TEMP_ZIP + os.sep + packdir + TESTPACK2
693 zi2 = zipimport.zipimporter(pkg_path)
694
695 # PEP 451
696 spec = zi2.find_spec(TESTMOD)
697 self.assertIsNotNone(spec)
698 self.assertIsInstance(spec.loader, zipimport.zipimporter)
699 self.assertFalse(spec.loader.is_package(TESTMOD))
700 load_mod = importlib.util.module_from_spec(spec)
701 spec.loader.exec_module(load_mod)
702 self.assertEqual(
703 spec.loader.get_filename(TESTMOD), load_mod.__file__)
704
705 mod_path = TESTPACK2 + os.sep + TESTMOD
706 mod_name = module_path_to_dotted_name(mod_path)
707 mod = importlib.import_module(mod_name)
708 self.assertTrue(mod_name in sys.modules)
709 self.assertIsNone(zi.get_source(TESTPACK2))
710 self.assertIsNone(zi.get_source(mod_path))
711 self.assertEqual(zi.get_filename(mod_path), mod.__file__)
712 # To pass in the module name instead of the path, we must use the
713 # right importer.
714 loader = mod.__loader__
715 self.assertIsNone(loader.get_source(mod_name))
716 self.assertEqual(loader.get_filename(mod_name), mod.__file__)
717
718 def testGetDataExplicitDirectories(self):
719 self.addCleanup(os_helper.unlink, TEMP_ZIP)
720 with ZipFile(TEMP_ZIP, 'w', compression=self.compression) as z:
721 z.mkdir('a')
722 z.mkdir('a/b')
723 z.mkdir('a/b/c')
724 data = bytes(range(256))
725 z.writestr('a/b/c/testdata.dat', data)
726 self._testGetData()
727
728 def testGetDataImplicitDirectories(self):
729 self.addCleanup(os_helper.unlink, TEMP_ZIP)
730 with ZipFile(TEMP_ZIP, 'w', compression=self.compression) as z:
731 data = bytes(range(256))
732 z.writestr('a/b/c/testdata.dat', data)
733 self._testGetData()
734
735 def _testGetData(self):
736 zi = zipimport.zipimporter(os.path.join(TEMP_ZIP, 'ignored'))
737 pathname = os.path.join('a', 'b', 'c', 'testdata.dat')
738 data = bytes(range(256))
739 self.assertEqual(zi.get_data(pathname), data)
740 self.assertEqual(zi.get_data(os.path.join(TEMP_ZIP, pathname)), data)
741 self.assertEqual(zi.get_data(os.path.join('a', 'b', '')), b'')
742 self.assertEqual(zi.get_data(os.path.join(TEMP_ZIP, 'a', 'b', '')), b'')
743 self.assertRaises(OSError, zi.get_data, os.path.join('a', 'b'))
744 self.assertRaises(OSError, zi.get_data, os.path.join(TEMP_ZIP, 'a', 'b'))
745
746 def testImporterAttr(self):
747 src = """if 1: # indent hack
748 def get_file():
749 return __file__
750 if __loader__.get_data("some.data") != b"some data":
751 raise AssertionError("bad data")\n"""
752 pyc = make_pyc(compile(src, "<???>", "exec"), NOW, len(src))
753 files = {TESTMOD + pyc_ext: pyc,
754 "some.data": "some data"}
755 self.doTest(pyc_ext, files, TESTMOD, prefix='')
756
757 def testDefaultOptimizationLevel(self):
758 # zipimport should use the default optimization level (#28131)
759 src = """if 1: # indent hack
760 def test(val):
761 assert(val)
762 return val\n"""
763 files = {TESTMOD + '.py': src}
764 self.makeZip(files)
765 sys.path.insert(0, TEMP_ZIP)
766 mod = importlib.import_module(TESTMOD)
767 self.assertEqual(mod.test(1), 1)
768 if __debug__:
769 self.assertRaises(AssertionError, mod.test, False)
770 else:
771 self.assertEqual(mod.test(0), 0)
772
773 def testImport_WithStuff(self):
774 # try importing from a zipfile which contains additional
775 # stuff at the beginning of the file
776 files = {TESTMOD + ".py": test_src}
777 self.doTest(".py", files, TESTMOD,
778 stuff=b"Some Stuff"*31)
779
780 def assertModuleSource(self, module):
781 self.assertEqual(inspect.getsource(module), test_src)
782
783 def testGetSource(self):
784 files = {TESTMOD + ".py": test_src}
785 self.doTest(".py", files, TESTMOD, call=self.assertModuleSource)
786
787 def testGetCompiledSource(self):
788 pyc = make_pyc(compile(test_src, "<???>", "exec"), NOW, len(test_src))
789 files = {TESTMOD + ".py": test_src,
790 TESTMOD + pyc_ext: pyc}
791 self.doTest(pyc_ext, files, TESTMOD, call=self.assertModuleSource)
792
793 def runDoctest(self, callback):
794 files = {TESTMOD + ".py": test_src,
795 "xyz.txt": ">>> log.append(True)\n"}
796 self.doTest(".py", files, TESTMOD, call=callback)
797
798 def doDoctestFile(self, module):
799 log = []
800 old_master, doctest.master = doctest.master, None
801 try:
802 doctest.testfile(
803 'xyz.txt', package=module, module_relative=True,
804 globs=locals()
805 )
806 finally:
807 doctest.master = old_master
808 self.assertEqual(log,[True])
809
810 def testDoctestFile(self):
811 self.runDoctest(self.doDoctestFile)
812
813 def doDoctestSuite(self, module):
814 log = []
815 doctest.DocFileTest(
816 'xyz.txt', package=module, module_relative=True,
817 globs=locals()
818 ).run()
819 self.assertEqual(log,[True])
820
821 def testDoctestSuite(self):
822 self.runDoctest(self.doDoctestSuite)
823
824 def doTraceback(self, module):
825 try:
826 module.do_raise()
827 except Exception as e:
828 tb = e.__traceback__.tb_next
829
830 f,lno,n,line = extract_tb(tb, 1)[0]
831 self.assertEqual(line, raise_src.strip())
832
833 f,lno,n,line = extract_stack(tb.tb_frame, 1)[0]
834 self.assertEqual(line, raise_src.strip())
835
836 s = io.StringIO()
837 print_tb(tb, 1, s)
838 self.assertEndsWith(s.getvalue(),
839 ' def do_raise(): raise TypeError\n'
840 '' if support.has_no_debug_ranges() else
841 ' ^^^^^^^^^^^^^^^\n'
842 )
843 else:
844 raise AssertionError("This ought to be impossible")
845
846 def testTraceback(self):
847 files = {TESTMOD + ".py": raise_src}
848 self.doTest(None, files, TESTMOD, call=self.doTraceback)
849
850 @unittest.skipIf(os_helper.TESTFN_UNENCODABLE is None,
851 "need an unencodable filename")
852 def testUnencodable(self):
853 filename = os_helper.TESTFN_UNENCODABLE + ".zip"
854 self.makeZip({TESTMOD + ".py": test_src}, filename)
855 spec = zipimport.zipimporter(filename).find_spec(TESTMOD)
856 mod = importlib.util.module_from_spec(spec)
857 spec.loader.exec_module(mod)
858
859 def testBytesPath(self):
860 filename = os_helper.TESTFN + ".zip"
861 self.makeZip({TESTMOD + ".py": test_src}, filename)
862
863 zipimport.zipimporter(filename)
864 with self.assertRaises(TypeError):
865 zipimport.zipimporter(os.fsencode(filename))
866 with self.assertRaises(TypeError):
867 zipimport.zipimporter(bytearray(os.fsencode(filename)))
868 with self.assertRaises(TypeError):
869 zipimport.zipimporter(memoryview(os.fsencode(filename)))
870
871 def testComment(self):
872 files = {TESTMOD + ".py": test_src}
873 self.doTest(".py", files, TESTMOD, comment=b"comment")
874
875 def testBeginningCruftAndComment(self):
876 files = {TESTMOD + ".py": test_src}
877 self.doTest(".py", files, TESTMOD, stuff=b"cruft" * 64, comment=b"hi")
878
879 def testLargestPossibleComment(self):
880 files = {TESTMOD + ".py": test_src}
881 self.doTest(".py", files, TESTMOD, comment=b"c" * ((1 << 16) - 1))
882
883 @support.requires_resource('cpu')
884 def testZip64(self):
885 files = self.getZip64Files()
886 self.doTest(".py", files, "f6")
887
888 @support.requires_resource('cpu')
889 def testZip64CruftAndComment(self):
890 files = self.getZip64Files()
891 self.doTest(".py", files, "f65536", comment=b"c" * ((1 << 16) - 1))
892
893 def testZip64LargeFile(self):
894 support.requires(
895 "largefile",
896 f"test generates files >{0xFFFFFFFF} bytes and takes a long time "
897 "to run"
898 )
899
900 # N.B.: We do a lot of gymnastics below in the ZIP_STORED case to save
901 # and reconstruct a sparse zip on systems that support sparse files.
902 # Instead of creating a ~8GB zip file mainly consisting of null bytes
903 # for every run of the test, we create the zip once and save off the
904 # non-null portions of the resulting file as data blobs with offsets
905 # that allow re-creating the zip file sparsely. This drops disk space
906 # usage to ~9KB for the ZIP_STORED case and drops that test time by ~2
907 # orders of magnitude. For the ZIP_DEFLATED case, however, we bite the
908 # bullet. The resulting zip file is ~8MB of non-null data; so the sparse
909 # trick doesn't work and would result in that full ~8MB zip data file
910 # being checked in to source control.
911 parts_glob = f"sparse-zip64-c{self.compression:d}-0x*.part"
912 full_parts_glob = os.path.join(TEST_DATA_DIR, parts_glob)
913 pre_built_zip_parts = glob.glob(full_parts_glob)
914
915 self.addCleanup(os_helper.unlink, TEMP_ZIP)
916 if not pre_built_zip_parts:
917 if self.compression != ZIP_STORED:
918 support.requires(
919 "cpu",
920 "test requires a lot of CPU for compression."
921 )
922 self.addCleanup(os_helper.unlink, os_helper.TESTFN)
923 with open(os_helper.TESTFN, "wb") as f:
924 f.write(b"data")
925 f.write(os.linesep.encode())
926 f.seek(0xffff_ffff, os.SEEK_CUR)
927 f.write(os.linesep.encode())
928 os.utime(os_helper.TESTFN, (0.0, 0.0))
929 with ZipFile(
930 TEMP_ZIP,
931 "w",
932 compression=self.compression,
933 strict_timestamps=False
934 ) as z:
935 z.write(os_helper.TESTFN, "data1")
936 z.writestr(
937 ZipInfo("module.py", (1980, 1, 1, 0, 0, 0)), test_src
938 )
939 z.write(os_helper.TESTFN, "data2")
940
941 # This "works" but relies on the zip format having a non-empty
942 # final page due to the trailing central directory to wind up with
943 # the correct length file.
944 def make_sparse_zip_parts(name):
945 empty_page = b"\0" * 4096
946 with open(name, "rb") as f:
947 part = None
948 try:
949 while True:
950 offset = f.tell()
951 data = f.read(len(empty_page))
952 if not data:
953 break
954 if data != empty_page:
955 if not part:
956 part_fullname = os.path.join(
957 TEST_DATA_DIR,
958 f"sparse-zip64-c{self.compression:d}-"
959 f"{offset:#011x}.part",
960 )
961 os.makedirs(
962 os.path.dirname(part_fullname),
963 exist_ok=True
964 )
965 part = open(part_fullname, "wb")
966 print("Created", part_fullname)
967 part.write(data)
968 else:
969 if part:
970 part.close()
971 part = None
972 finally:
973 if part:
974 part.close()
975
976 if self.compression == ZIP_STORED:
977 print(f"Creating sparse parts to check in into {TEST_DATA_DIR}:")
978 make_sparse_zip_parts(TEMP_ZIP)
979
980 else:
981 def extract_offset(name):
982 if m := re.search(r"-(0x[0-9a-f]{9})\.part$", name):
983 return int(m.group(1), base=16)
984 raise ValueError(f"{name=} does not fit expected pattern.")
985 offset_parts = [(extract_offset(n), n) for n in pre_built_zip_parts]
986 with open(TEMP_ZIP, "wb") as f:
987 for offset, part_fn in sorted(offset_parts):
988 with open(part_fn, "rb") as part:
989 f.seek(offset, os.SEEK_SET)
990 f.write(part.read())
991 # Confirm that the reconstructed zip file works and looks right.
992 with ZipFile(TEMP_ZIP, "r") as z:
993 self.assertEqual(
994 z.getinfo("module.py").date_time, (1980, 1, 1, 0, 0, 0)
995 )
996 self.assertEqual(
997 z.read("module.py"), test_src.encode(),
998 msg=f"Recreate {full_parts_glob}, unexpected contents."
999 )
1000 def assertDataEntry(name):
1001 zinfo = z.getinfo(name)
1002 self.assertEqual(zinfo.date_time, (1980, 1, 1, 0, 0, 0))
1003 self.assertGreater(zinfo.file_size, 0xffff_ffff)
1004 assertDataEntry("data1")
1005 assertDataEntry("data2")
1006
1007 self.doTestWithPreBuiltZip(".py", "module")
1008
1009
1010 @support.requires_zlib()
1011 class CompressedZipImportTestCase(UncompressedZipImportTestCase):
1012 compression = ZIP_DEFLATED
1013
1014
1015 class BadFileZipImportTestCase(unittest.TestCase):
1016 def assertZipFailure(self, filename):
1017 self.assertRaises(zipimport.ZipImportError,
1018 zipimport.zipimporter, filename)
1019
1020 def testNoFile(self):
1021 self.assertZipFailure('AdfjdkFJKDFJjdklfjs')
1022
1023 def testEmptyFilename(self):
1024 self.assertZipFailure('')
1025
1026 def testBadArgs(self):
1027 self.assertRaises(TypeError, zipimport.zipimporter, None)
1028 self.assertRaises(TypeError, zipimport.zipimporter, TESTMOD, kwd=None)
1029 self.assertRaises(TypeError, zipimport.zipimporter,
1030 list(os.fsencode(TESTMOD)))
1031
1032 def testFilenameTooLong(self):
1033 self.assertZipFailure('A' * 33000)
1034
1035 def testEmptyFile(self):
1036 os_helper.unlink(TESTMOD)
1037 os_helper.create_empty_file(TESTMOD)
1038 self.assertZipFailure(TESTMOD)
1039
1040 @unittest.skipIf(support.is_wasi, "mode 000 not supported.")
1041 def testFileUnreadable(self):
1042 os_helper.unlink(TESTMOD)
1043 fd = os.open(TESTMOD, os.O_CREAT, 000)
1044 try:
1045 os.close(fd)
1046
1047 with self.assertRaises(zipimport.ZipImportError) as cm:
1048 zipimport.zipimporter(TESTMOD)
1049 finally:
1050 # If we leave "the read-only bit" set on Windows, nothing can
1051 # delete TESTMOD, and later tests suffer bogus failures.
1052 os.chmod(TESTMOD, 0o666)
1053 os_helper.unlink(TESTMOD)
1054
1055 def testNotZipFile(self):
1056 os_helper.unlink(TESTMOD)
1057 fp = open(TESTMOD, 'w+')
1058 fp.write('a' * 22)
1059 fp.close()
1060 self.assertZipFailure(TESTMOD)
1061
1062 # XXX: disabled until this works on Big-endian machines
1063 def _testBogusZipFile(self):
1064 os_helper.unlink(TESTMOD)
1065 fp = open(TESTMOD, 'w+')
1066 fp.write(struct.pack('=I', 0x06054B50))
1067 fp.write('a' * 18)
1068 fp.close()
1069 z = zipimport.zipimporter(TESTMOD)
1070
1071 try:
1072 with warnings.catch_warnings():
1073 warnings.simplefilter("ignore", DeprecationWarning)
1074 self.assertRaises(TypeError, z.load_module, None)
1075 self.assertRaises(TypeError, z.find_module, None)
1076 self.assertRaises(TypeError, z.find_spec, None)
1077 self.assertRaises(TypeError, z.exec_module, None)
1078 self.assertRaises(TypeError, z.is_package, None)
1079 self.assertRaises(TypeError, z.get_code, None)
1080 self.assertRaises(TypeError, z.get_data, None)
1081 self.assertRaises(TypeError, z.get_source, None)
1082
1083 error = zipimport.ZipImportError
1084 self.assertIsNone(z.find_spec('abc'))
1085
1086 with warnings.catch_warnings():
1087 warnings.simplefilter("ignore", DeprecationWarning)
1088 self.assertRaises(error, z.load_module, 'abc')
1089 self.assertRaises(error, z.get_code, 'abc')
1090 self.assertRaises(OSError, z.get_data, 'abc')
1091 self.assertRaises(error, z.get_source, 'abc')
1092 self.assertRaises(error, z.is_package, 'abc')
1093 finally:
1094 zipimport._zip_directory_cache.clear()
1095
1096
1097 def tearDownModule():
1098 os_helper.unlink(TESTMOD)
1099
1100
1101 if __name__ == "__main__":
1102 unittest.main()