Debian Codegraph @

lldb/packages/Python/lldbsuite/test/test_result.py

raw
1 """
2 Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
3 See https://llvm.org/LICENSE.txt for license information.
4 SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
5
6 Provides the LLDBTestResult class, which holds information about progress
7 and results of a single test run.
8 """
9
10 # System modules
11 import os
12 import traceback
13
14 # Third-party modules
15 import unittest
16
17 # LLDB Modules
18 from . import configuration
19 from lldbsuite.test_event import build_exception
20
21
22 class LLDBTestResult(unittest.TextTestResult):
23 """
24 Enforce a singleton pattern to allow introspection of test progress.
25
26 Overwrite addError(), addFailure(), and addExpectedFailure() methods
27 to enable each test instance to track its failure/error status. It
28 is used in the LLDB test framework to emit detailed trace messages
29 to a log file for easier human inspection of test failures/errors.
30 """
31
32 __singleton__ = None
33 __ignore_singleton__ = False
34
35 @staticmethod
36 def getTerminalSize():
37 import os
38
39 env = os.environ
40
41 def ioctl_GWINSZ(fd):
42 try:
43 import fcntl
44 import termios
45 import struct
46
47 cr = struct.unpack("hh", fcntl.ioctl(fd, termios.TIOCGWINSZ, "1234"))
48 except:
49 return
50 return cr
51
52 cr = ioctl_GWINSZ(0) or ioctl_GWINSZ(1) or ioctl_GWINSZ(2)
53 if not cr:
54 try:
55 fd = os.open(os.ctermid(), os.O_RDONLY)
56 cr = ioctl_GWINSZ(fd)
57 os.close(fd)
58 except:
59 pass
60 if not cr:
61 cr = (env.get("LINES", 25), env.get("COLUMNS", 80))
62 return int(cr[1]), int(cr[0])
63
64 def __init__(self, *args):
65 if not LLDBTestResult.__ignore_singleton__ and LLDBTestResult.__singleton__:
66 raise Exception("LLDBTestResult instantiated more than once")
67 super(LLDBTestResult, self).__init__(*args)
68 LLDBTestResult.__singleton__ = self
69 # Now put this singleton into the lldb module namespace.
70 configuration.test_result = self
71 # Computes the format string for displaying the counter.
72 counterWidth = len(str(configuration.suite.countTestCases()))
73 self.fmt = "%" + str(counterWidth) + "d: "
74 self.indentation = " " * (counterWidth + 2)
75 # This counts from 1 .. suite.countTestCases().
76 self.counter = 0
77 (width, height) = LLDBTestResult.getTerminalSize()
78
79 def _config_string(self, test):
80 compiler = getattr(test, "getCompiler", None)
81 arch = getattr(test, "getArchitecture", None)
82 return "%s-%s" % (compiler() if compiler else "", arch() if arch else "")
83
84 def _exc_info_to_string(self, err, test):
85 """Overrides superclass TestResult's method in order to append
86 our test config info string to the exception info string."""
87 if hasattr(test, "getArchitecture") and hasattr(test, "getCompiler"):
88 return "%sConfig=%s-%s" % (
89 super(LLDBTestResult, self)._exc_info_to_string(err, test),
90 test.getArchitecture(),
91 test.getCompiler(),
92 )
93 else:
94 return super(LLDBTestResult, self)._exc_info_to_string(err, test)
95
96 def getDescription(self, test):
97 doc_first_line = test.shortDescription()
98 if self.descriptions and doc_first_line:
99 return "\n".join((str(test), self.indentation + doc_first_line))
100 else:
101 return str(test)
102
103 def _getTestPath(self, test):
104 # Use test.test_filename if the test was created with
105 # lldbinline.MakeInlineTest().
106 if test is None:
107 return ""
108 elif hasattr(test, "test_filename"):
109 return test.test_filename
110 else:
111 import inspect
112
113 return inspect.getsourcefile(test.__class__)
114
115 def _getFileBasedCategories(self, test):
116 """
117 Returns the list of categories to which this test case belongs by
118 collecting values of "categories" files. We start at the folder the test is in
119 and traverse the hierarchy upwards until the test-suite root directory.
120 """
121 start_path = self._getTestPath(test)
122
123 import os.path
124
125 folder = os.path.dirname(start_path)
126
127 from lldbsuite import lldb_test_root as test_root
128
129 if test_root != os.path.commonprefix([folder, test_root]):
130 raise Exception(
131 "The test file %s is outside the test root directory" % start_path
132 )
133
134 categories = set()
135 while not os.path.samefile(folder, test_root):
136 categories_file_name = os.path.join(folder, "categories")
137 if os.path.exists(categories_file_name):
138 categories_file = open(categories_file_name, "r")
139 categories_str = categories_file.readline().strip()
140 categories_file.close()
141 categories.update(categories_str.split(","))
142 folder = os.path.dirname(folder)
143
144 return list(categories)
145
146 def getCategoriesForTest(self, test):
147 """
148 Gets all the categories for the currently running test method in test case
149 """
150 test_categories = []
151 test_categories.extend(getattr(test, "categories", []))
152
153 test_method = getattr(test, test._testMethodName)
154 if test_method is not None:
155 test_categories.extend(getattr(test_method, "categories", []))
156
157 test_categories.extend(self._getFileBasedCategories(test))
158
159 return test_categories
160
161 def hardMarkAsSkipped(self, test):
162 getattr(test, test._testMethodName).__func__.__unittest_skip__ = True
163 getattr(
164 test, test._testMethodName
165 ).__func__.__unittest_skip_why__ = (
166 "test case does not fall in any category of interest for this run"
167 )
168
169 def checkExclusion(self, exclusion_list, name):
170 if exclusion_list:
171 import re
172
173 for item in exclusion_list:
174 if re.search(item, name):
175 return True
176 return False
177
178 def checkCategoryExclusion(self, exclusion_list, test):
179 return not set(exclusion_list).isdisjoint(self.getCategoriesForTest(test))
180
181 def startTest(self, test):
182 if configuration.shouldSkipBecauseOfCategories(self.getCategoriesForTest(test)):
183 self.hardMarkAsSkipped(test)
184 if self.checkExclusion(configuration.skip_tests, test.id()):
185 self.hardMarkAsSkipped(test)
186
187 self.counter += 1
188 test.test_number = self.counter
189 if self.showAll:
190 self.stream.write(self.fmt % self.counter)
191 super(LLDBTestResult, self).startTest(test)
192
193 def addSuccess(self, test):
194 if self.checkExclusion(
195 configuration.xfail_tests, test.id()
196 ) or self.checkCategoryExclusion(configuration.xfail_categories, test):
197 self.addUnexpectedSuccess(test, None)
198 return
199
200 super(LLDBTestResult, self).addSuccess(test)
201 self.stream.write(
202 "PASS: LLDB (%s) :: %s\n" % (self._config_string(test), str(test))
203 )
204
205 def _isBuildError(self, err_tuple):
206 exception = err_tuple[1]
207 return isinstance(exception, build_exception.BuildError)
208
209 def _saveBuildErrorTuple(self, test, err):
210 # Adjust the error description so it prints the build command and build error
211 # rather than an uninformative Python backtrace.
212 build_error = err[1]
213 error_description = "{}\nTest Directory:\n{}".format(
214 str(build_error), os.path.dirname(self._getTestPath(test))
215 )
216 self.errors.append((test, error_description))
217 self._mirrorOutput = True
218
219 def addError(self, test, err):
220 configuration.sdir_has_content = True
221 if self._isBuildError(err):
222 self._saveBuildErrorTuple(test, err)
223 else:
224 super(LLDBTestResult, self).addError(test, err)
225
226 method = getattr(test, "markError", None)
227 if method:
228 method()
229 self.stream.write(
230 "FAIL: LLDB (%s) :: %s\n" % (self._config_string(test), str(test))
231 )
232
233 def addCleanupError(self, test, err):
234 configuration.sdir_has_content = True
235 super(LLDBTestResult, self).addCleanupError(test, err)
236 method = getattr(test, "markCleanupError", None)
237 if method:
238 method()
239 self.stream.write(
240 "CLEANUP ERROR: LLDB (%s) :: %s\n%s\n"
241 % (self._config_string(test), str(test), traceback.format_exc())
242 )
243
244 def addFailure(self, test, err):
245 if self.checkExclusion(
246 configuration.xfail_tests, test.id()
247 ) or self.checkCategoryExclusion(configuration.xfail_categories, test):
248 self.addExpectedFailure(test, err)
249 return
250
251 configuration.sdir_has_content = True
252 super(LLDBTestResult, self).addFailure(test, err)
253 method = getattr(test, "markFailure", None)
254 if method:
255 method()
256 self.stream.write(
257 "FAIL: LLDB (%s) :: %s\n" % (self._config_string(test), str(test))
258 )
259 if configuration.use_categories:
260 test_categories = self.getCategoriesForTest(test)
261 for category in test_categories:
262 if category in configuration.failures_per_category:
263 configuration.failures_per_category[category] = (
264 configuration.failures_per_category[category] + 1
265 )
266 else:
267 configuration.failures_per_category[category] = 1
268
269 def addExpectedFailure(self, test, err):
270 configuration.sdir_has_content = True
271 super(LLDBTestResult, self).addExpectedFailure(test, err)
272 method = getattr(test, "markExpectedFailure", None)
273 if method:
274 method(err)
275 self.stream.write(
276 "XFAIL: LLDB (%s) :: %s\n" % (self._config_string(test), str(test))
277 )
278
279 def addSkip(self, test, reason):
280 configuration.sdir_has_content = True
281 super(LLDBTestResult, self).addSkip(test, reason)
282 method = getattr(test, "markSkippedTest", None)
283 if method:
284 method()
285 self.stream.write(
286 "UNSUPPORTED: LLDB (%s) :: %s (%s) \n"
287 % (self._config_string(test), str(test), reason)
288 )
289
290 def addUnexpectedSuccess(self, test):
291 configuration.sdir_has_content = True
292 super(LLDBTestResult, self).addUnexpectedSuccess(test)
293 method = getattr(test, "markUnexpectedSuccess", None)
294 if method:
295 method()
296 self.stream.write(
297 "XPASS: LLDB (%s) :: %s\n" % (self._config_string(test), str(test))
298 )