Debian Codegraph @

Lib/test/libregrtest/refleak.py

raw
1 import os
2 import sys
3 import warnings
4 from inspect import isabstract
5 from typing import Any
6 import linecache
7
8 from test import support
9 from test.support import os_helper
10 from test.support import refleak_helper
11
12 from .runtests import HuntRefleak
13 from .utils import clear_caches
14
15 try:
16 from _abc import _get_dump
17 except ImportError:
18 import weakref
19
20 def _get_dump(cls):
21 # Reimplement _get_dump() for pure-Python implementation of
22 # the abc module (Lib/_py_abc.py)
23 registry_weakrefs = set(weakref.ref(obj) for obj in cls._abc_registry)
24 return (registry_weakrefs, cls._abc_cache,
25 cls._abc_negative_cache, cls._abc_negative_cache_version)
26
27
28 def save_support_xml(filename):
29 if support.junit_xml_list is None:
30 return
31
32 import pickle
33 with open(filename, 'xb') as fp:
34 pickle.dump(support.junit_xml_list, fp)
35 support.junit_xml_list = None
36
37
38 def restore_support_xml(filename):
39 try:
40 fp = open(filename, 'rb')
41 except FileNotFoundError:
42 return
43
44 import pickle
45 with fp:
46 xml_list = pickle.load(fp)
47 os.unlink(filename)
48
49 support.junit_xml_list = xml_list
50
51
52 def runtest_refleak(test_name, test_func,
53 hunt_refleak: HuntRefleak,
54 quiet: bool):
55 """Run a test multiple times, looking for reference leaks.
56
57 Returns:
58 False if the test didn't leak references; True if we detected refleaks.
59 """
60 # This code is hackish and inelegant, but it seems to do the job.
61 import copyreg
62 import collections.abc
63
64 if not hasattr(sys, 'gettotalrefcount'):
65 raise Exception("Tracking reference leaks requires a debug build "
66 "of Python")
67
68 # Avoid false positives due to various caches
69 # filling slowly with random data:
70 warm_caches()
71
72 # Save current values for dash_R_cleanup() to restore.
73 fs = warnings.filters[:]
74 ps = copyreg.dispatch_table.copy()
75 pic = sys.path_importer_cache.copy()
76 zdc: dict[str, Any] | None
77 # Linecache holds a cache with the source of interactive code snippets
78 # (e.g. code typed in the REPL). This cache is not cleared by
79 # linecache.clearcache(). We need to save and restore it to avoid false
80 # positives.
81 linecache_data = linecache.cache.copy(), linecache._interactive_cache.copy() # type: ignore[attr-defined]
82 try:
83 import zipimport
84 except ImportError:
85 zdc = None # Run unmodified on platforms without zipimport support
86 else:
87 # private attribute that mypy doesn't know about:
88 zdc = zipimport._zip_directory_cache.copy() # type: ignore[attr-defined]
89 abcs = {}
90 for abc in [getattr(collections.abc, a) for a in collections.abc.__all__]:
91 if not isabstract(abc):
92 continue
93 for obj in abc.__subclasses__() + [abc]:
94 abcs[obj] = _get_dump(obj)[0]
95
96 # bpo-31217: Integer pool to get a single integer object for the same
97 # value. The pool is used to prevent false alarm when checking for memory
98 # block leaks. Fill the pool with values in -1000..1000 which are the most
99 # common (reference, memory block, file descriptor) differences.
100 int_pool = {value: value for value in range(-1000, 1000)}
101 def get_pooled_int(value):
102 return int_pool.setdefault(value, value)
103
104 warmups = hunt_refleak.warmups
105 runs = hunt_refleak.runs
106 filename = hunt_refleak.filename
107 repcount = warmups + runs
108
109 # Pre-allocate to ensure that the loop doesn't allocate anything new
110 rep_range = list(range(repcount))
111 rc_deltas = [0] * repcount
112 alloc_deltas = [0] * repcount
113 fd_deltas = [0] * repcount
114 getallocatedblocks = sys.getallocatedblocks
115 gettotalrefcount = sys.gettotalrefcount
116 getunicodeinternedsize = sys.getunicodeinternedsize
117 fd_count = os_helper.fd_count
118 # initialize variables to make pyflakes quiet
119 rc_before = alloc_before = fd_before = interned_immortal_before = 0
120
121 if not quiet:
122 print("beginning", repcount, "repetitions. Showing number of leaks "
123 "(. for 0 or less, X for 10 or more)",
124 file=sys.stderr)
125 numbers = ("1234567890"*(repcount//10 + 1))[:repcount]
126 numbers = numbers[:warmups] + ':' + numbers[warmups:]
127 print(numbers, file=sys.stderr, flush=True)
128
129 xml_filename = 'refleak-xml.tmp'
130 result = None
131 dash_R_cleanup(fs, ps, pic, zdc, abcs, linecache_data)
132
133 for i in rep_range:
134 support.gc_collect()
135 current = refleak_helper._hunting_for_refleaks
136 refleak_helper._hunting_for_refleaks = True
137 try:
138 result = test_func()
139 finally:
140 refleak_helper._hunting_for_refleaks = current
141
142 save_support_xml(xml_filename)
143 dash_R_cleanup(fs, ps, pic, zdc, abcs, linecache_data)
144 support.gc_collect()
145
146 # Read memory statistics immediately after the garbage collection.
147 # Also, readjust the reference counts and alloc blocks by ignoring
148 # any strings that might have been interned during test_func. These
149 # strings will be deallocated at runtime shutdown
150 interned_immortal_after = getunicodeinternedsize(
151 # Use an internal-only keyword argument that mypy doesn't know yet
152 _only_immortal=True) # type: ignore[call-arg]
153 alloc_after = getallocatedblocks() - interned_immortal_after
154 rc_after = gettotalrefcount()
155 fd_after = fd_count()
156
157 rc_deltas[i] = get_pooled_int(rc_after - rc_before)
158 alloc_deltas[i] = get_pooled_int(alloc_after - alloc_before)
159 fd_deltas[i] = get_pooled_int(fd_after - fd_before)
160
161 if not quiet:
162 # use max, not sum, so total_leaks is one of the pooled ints
163 total_leaks = max(rc_deltas[i], alloc_deltas[i], fd_deltas[i])
164 if total_leaks <= 0:
165 symbol = '.'
166 elif total_leaks < 10:
167 symbol = (
168 '.', '1', '2', '3', '4', '5', '6', '7', '8', '9',
169 )[total_leaks]
170 else:
171 symbol = 'X'
172 if i == warmups:
173 print(' ', end='', file=sys.stderr, flush=True)
174 print(symbol, end='', file=sys.stderr, flush=True)
175 del total_leaks
176 del symbol
177
178 alloc_before = alloc_after
179 rc_before = rc_after
180 fd_before = fd_after
181 interned_immortal_before = interned_immortal_after
182
183 restore_support_xml(xml_filename)
184
185 if not quiet:
186 print(file=sys.stderr)
187
188 # These checkers return False on success, True on failure
189 def check_rc_deltas(deltas):
190 # Checker for reference counters and memory blocks.
191 #
192 # bpo-30776: Try to ignore false positives:
193 #
194 # [3, 0, 0]
195 # [0, 1, 0]
196 # [8, -8, 1]
197 #
198 # Expected leaks:
199 #
200 # [5, 5, 6]
201 # [10, 1, 1]
202 return all(delta >= 1 for delta in deltas)
203
204 def check_fd_deltas(deltas):
205 return any(deltas)
206
207 failed = False
208 for deltas, item_name, checker in [
209 (rc_deltas, 'references', check_rc_deltas),
210 (alloc_deltas, 'memory blocks', check_rc_deltas),
211 (fd_deltas, 'file descriptors', check_fd_deltas)
212 ]:
213 # ignore warmup runs
214 deltas = deltas[warmups:]
215 failing = checker(deltas)
216 suspicious = any(deltas)
217 if failing or suspicious:
218 msg = '%s leaked %s %s, sum=%s' % (
219 test_name, deltas, item_name, sum(deltas))
220 print(msg, end='', file=sys.stderr)
221 if failing:
222 print(file=sys.stderr, flush=True)
223 with open(filename, "a", encoding="utf-8") as refrep:
224 print(msg, file=refrep)
225 refrep.flush()
226 failed = True
227 else:
228 print(' (this is fine)', file=sys.stderr, flush=True)
229 return (failed, result)
230
231
232 def dash_R_cleanup(fs, ps, pic, zdc, abcs, linecache_data):
233 import copyreg
234 import collections.abc
235
236 # Restore some original values.
237 warnings.filters[:] = fs
238 copyreg.dispatch_table.clear()
239 copyreg.dispatch_table.update(ps)
240 sys.path_importer_cache.clear()
241 sys.path_importer_cache.update(pic)
242 lcache, linteractive = linecache_data
243 linecache._interactive_cache.clear()
244 linecache._interactive_cache.update(linteractive)
245 linecache.cache.clear()
246 linecache.cache.update(lcache)
247 try:
248 import zipimport
249 except ImportError:
250 pass # Run unmodified on platforms without zipimport support
251 else:
252 zipimport._zip_directory_cache.clear()
253 zipimport._zip_directory_cache.update(zdc)
254
255 # Clear ABC registries, restoring previously saved ABC registries.
256 abs_classes = [getattr(collections.abc, a) for a in collections.abc.__all__]
257 abs_classes = filter(isabstract, abs_classes)
258 for abc in abs_classes:
259 for obj in abc.__subclasses__() + [abc]:
260 refs = abcs.get(obj, None)
261 if refs is not None:
262 obj._abc_registry_clear()
263 for ref in refs:
264 subclass = ref()
265 if subclass is not None:
266 obj.register(subclass)
267 obj._abc_caches_clear()
268
269 # Clear caches
270 clear_caches()
271
272 # Clear other caches last (previous function calls can re-populate them):
273 sys._clear_internal_caches()
274
275
276 def warm_caches() -> None:
277 # char cache
278 s = bytes(range(256))
279 for i in range(256):
280 s[i:i+1]
281 # unicode cache
282 [chr(i) for i in range(256)]
283 # int cache
284 list(range(-5, 257))