| 1 |
import re |
| 2 |
import textwrap |
| 3 |
import unittest |
| 4 |
|
| 5 |
|
| 6 |
from test import support |
| 7 |
from test.support import import_helper, requires_subprocess |
| 8 |
from test.support.script_helper import assert_python_failure, assert_python_ok |
| 9 |
|
| 10 |
|
| 11 |
# Skip this test if the _testcapi and _testinternalcapi extensions are not |
| 12 |
# available. |
| 13 |
_testcapi = import_helper.import_module('_testcapi') |
| 14 |
_testinternalcapi = import_helper.import_module('_testinternalcapi') |
| 15 |
|
| 16 |
@requires_subprocess() |
| 17 |
class PyMemDebugTests(unittest.TestCase): |
| 18 |
PYTHONMALLOC = 'debug' |
| 19 |
# '0x04c06e0' or '04C06E0' |
| 20 |
PTR_REGEX = r'(?:0x)?[0-9a-fA-F]+' |
| 21 |
|
| 22 |
def check(self, code): |
| 23 |
with support.SuppressCrashReport(): |
| 24 |
out = assert_python_failure( |
| 25 |
'-c', code, |
| 26 |
PYTHONMALLOC=self.PYTHONMALLOC, |
| 27 |
# FreeBSD: instruct jemalloc to not fill freed() memory |
| 28 |
# with junk byte 0x5a, see JEMALLOC(3) |
| 29 |
MALLOC_CONF="junk:false", |
| 30 |
) |
| 31 |
stderr = out.err |
| 32 |
return stderr.decode('ascii', 'replace') |
| 33 |
|
| 34 |
def test_buffer_overflow(self): |
| 35 |
out = self.check('import _testcapi; _testcapi.pymem_buffer_overflow()') |
| 36 |
regex = (r"Debug memory block at address p={ptr}: API 'm'\n" |
| 37 |
r" 16 bytes originally requested\n" |
| 38 |
r" The [0-9] pad bytes at p-[0-9] are FORBIDDENBYTE, as expected.\n" |
| 39 |
r" The [0-9] pad bytes at tail={ptr} are not all FORBIDDENBYTE \(0x[0-9a-f]{{2}}\):\n" |
| 40 |
r" at tail\+0: 0x78 \*\*\* OUCH\n" |
| 41 |
r" at tail\+1: 0xfd\n" |
| 42 |
r" at tail\+2: 0xfd\n" |
| 43 |
r" .*\n" |
| 44 |
r"( The block was made by call #[0-9]+ to debug malloc/realloc.\n)?" |
| 45 |
r" Data at p: cd cd cd .*\n" |
| 46 |
r"\n" |
| 47 |
r"Enable tracemalloc to get the memory block allocation traceback\n" |
| 48 |
r"\n" |
| 49 |
r"Fatal Python error: _PyMem_DebugRawFree: bad trailing pad byte") |
| 50 |
regex = regex.format(ptr=self.PTR_REGEX) |
| 51 |
regex = re.compile(regex, flags=re.DOTALL) |
| 52 |
self.assertRegex(out, regex) |
| 53 |
|
| 54 |
def test_api_misuse(self): |
| 55 |
out = self.check('import _testcapi; _testcapi.pymem_api_misuse()') |
| 56 |
regex = (r"Debug memory block at address p={ptr}: API 'm'\n" |
| 57 |
r" 16 bytes originally requested\n" |
| 58 |
r" The [0-9] pad bytes at p-[0-9] are FORBIDDENBYTE, as expected.\n" |
| 59 |
r" The [0-9] pad bytes at tail={ptr} are FORBIDDENBYTE, as expected.\n" |
| 60 |
r"( The block was made by call #[0-9]+ to debug malloc/realloc.\n)?" |
| 61 |
r" Data at p: cd cd cd .*\n" |
| 62 |
r"\n" |
| 63 |
r"Enable tracemalloc to get the memory block allocation traceback\n" |
| 64 |
r"\n" |
| 65 |
r"Fatal Python error: _PyMem_DebugRawFree: bad ID: Allocated using API 'm', verified using API 'r'\n") |
| 66 |
regex = regex.format(ptr=self.PTR_REGEX) |
| 67 |
self.assertRegex(out, regex) |
| 68 |
|
| 69 |
def check_malloc_without_gil(self, code): |
| 70 |
out = self.check(code) |
| 71 |
if not support.Py_GIL_DISABLED: |
| 72 |
expected = ('Fatal Python error: _PyMem_DebugMalloc: ' |
| 73 |
'Python memory allocator called without holding the GIL') |
| 74 |
else: |
| 75 |
expected = ('Fatal Python error: _PyMem_DebugMalloc: ' |
| 76 |
'Python memory allocator called without an active thread state. ' |
| 77 |
'Are you trying to call it inside of a Py_BEGIN_ALLOW_THREADS block?') |
| 78 |
self.assertIn(expected, out) |
| 79 |
|
| 80 |
def test_pymem_malloc_without_gil(self): |
| 81 |
# Debug hooks must raise an error if PyMem_Malloc() is called |
| 82 |
# without holding the GIL |
| 83 |
code = 'import _testcapi; _testcapi.pymem_malloc_without_gil()' |
| 84 |
self.check_malloc_without_gil(code) |
| 85 |
|
| 86 |
def test_pyobject_malloc_without_gil(self): |
| 87 |
# Debug hooks must raise an error if PyObject_Malloc() is called |
| 88 |
# without holding the GIL |
| 89 |
code = 'import _testcapi; _testcapi.pyobject_malloc_without_gil()' |
| 90 |
self.check_malloc_without_gil(code) |
| 91 |
|
| 92 |
def check_pyobject_is_freed(self, func_name): |
| 93 |
code = textwrap.dedent(f''' |
| 94 |
import gc, os, sys, _testinternalcapi |
| 95 |
# Disable the GC to avoid crash on GC collection |
| 96 |
gc.disable() |
| 97 |
_testinternalcapi.{func_name}() |
| 98 |
# Exit immediately to avoid a crash while deallocating |
| 99 |
# the invalid object |
| 100 |
os._exit(0) |
| 101 |
''') |
| 102 |
assert_python_ok( |
| 103 |
'-c', code, |
| 104 |
PYTHONMALLOC=self.PYTHONMALLOC, |
| 105 |
MALLOC_CONF="junk:false", |
| 106 |
) |
| 107 |
|
| 108 |
def test_pyobject_null_is_freed(self): |
| 109 |
self.check_pyobject_is_freed('check_pyobject_null_is_freed') |
| 110 |
|
| 111 |
def test_pyobject_uninitialized_is_freed(self): |
| 112 |
self.check_pyobject_is_freed('check_pyobject_uninitialized_is_freed') |
| 113 |
|
| 114 |
def test_pyobject_forbidden_bytes_is_freed(self): |
| 115 |
self.check_pyobject_is_freed('check_pyobject_forbidden_bytes_is_freed') |
| 116 |
|
| 117 |
def test_pyobject_freed_is_freed(self): |
| 118 |
self.check_pyobject_is_freed('check_pyobject_freed_is_freed') |
| 119 |
|
| 120 |
# Python built with Py_TRACE_REFS fail with a fatal error in |
| 121 |
# _PyRefchain_Trace() on memory allocation error. |
| 122 |
@unittest.skipIf(support.Py_TRACE_REFS, 'cannot test Py_TRACE_REFS build') |
| 123 |
def test_set_nomemory(self): |
| 124 |
code = """if 1: |
| 125 |
import _testcapi |
| 126 |
|
| 127 |
class C(): pass |
| 128 |
|
| 129 |
# The first loop tests both functions and that remove_mem_hooks() |
| 130 |
# can be called twice in a row. The second loop checks a call to |
| 131 |
# set_nomemory() after a call to remove_mem_hooks(). The third |
| 132 |
# loop checks the start and stop arguments of set_nomemory(). |
| 133 |
for outer_cnt in range(1, 4): |
| 134 |
start = 10 * outer_cnt |
| 135 |
for j in range(100): |
| 136 |
if j == 0: |
| 137 |
if outer_cnt != 3: |
| 138 |
_testcapi.set_nomemory(start) |
| 139 |
else: |
| 140 |
_testcapi.set_nomemory(start, start + 1) |
| 141 |
try: |
| 142 |
C() |
| 143 |
except MemoryError as e: |
| 144 |
if outer_cnt != 3: |
| 145 |
_testcapi.remove_mem_hooks() |
| 146 |
print('MemoryError', outer_cnt, j) |
| 147 |
_testcapi.remove_mem_hooks() |
| 148 |
break |
| 149 |
""" |
| 150 |
rc, out, err = assert_python_ok('-c', code) |
| 151 |
lines = out.splitlines() |
| 152 |
for i, line in enumerate(lines, 1): |
| 153 |
self.assertIn(b'MemoryError', out) |
| 154 |
*_, count = line.split(b' ') |
| 155 |
count = int(count) |
| 156 |
self.assertLessEqual(count, i*10) |
| 157 |
self.assertGreaterEqual(count, i*10-4) |
| 158 |
|
| 159 |
|
| 160 |
# free-threading requires mimalloc (not malloc) |
| 161 |
@support.requires_gil_enabled() |
| 162 |
class PyMemMallocDebugTests(PyMemDebugTests): |
| 163 |
PYTHONMALLOC = 'malloc_debug' |
| 164 |
|
| 165 |
|
| 166 |
@unittest.skipUnless(support.with_pymalloc(), 'need pymalloc') |
| 167 |
class PyMemPymallocDebugTests(PyMemDebugTests): |
| 168 |
PYTHONMALLOC = 'pymalloc_debug' |
| 169 |
|
| 170 |
|
| 171 |
@unittest.skipUnless(support.with_mimalloc(), 'need mimaloc') |
| 172 |
class PyMemMimallocDebugTests(PyMemDebugTests): |
| 173 |
PYTHONMALLOC = 'mimalloc_debug' |
| 174 |
|
| 175 |
|
| 176 |
@unittest.skipUnless(support.Py_DEBUG, 'need Py_DEBUG') |
| 177 |
class PyMemDefaultTests(PyMemDebugTests): |
| 178 |
# test default allocator of Python compiled in debug mode |
| 179 |
PYTHONMALLOC = '' |
| 180 |
|
| 181 |
|
| 182 |
if __name__ == "__main__": |
| 183 |
unittest.main() |