Debian Codegraph @

third_party/spirv-tools/test/tools/spirv_test_framework.py

raw
1 # Copyright (c) 2018 Google LLC
2 #
3 # Licensed under the Apache License, Version 2.0 (the "License");
4 # you may not use this file except in compliance with the License.
5 # You may obtain a copy of the License at
6 #
7 # http://www.apache.org/licenses/LICENSE-2.0
8 #
9 # Unless required by applicable law or agreed to in writing, software
10 # distributed under the License is distributed on an "AS IS" BASIS,
11 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 # See the License for the specific language governing permissions and
13 # limitations under the License.
14 """Manages and runs tests from the current working directory.
15
16 This will traverse the current working directory and look for python files that
17 contain subclasses of SpirvTest.
18
19 If a class has an @inside_spirv_testsuite decorator, an instance of that
20 class will be created and serve as a test case in that testsuite. The test
21 case is then run by the following steps:
22
23 1. A temporary directory will be created.
24 2. The spirv_args member variable will be inspected and all placeholders in it
25 will be expanded by calling instantiate_for_spirv_args() on placeholders.
26 The transformed list elements are then supplied as arguments to the spirv-*
27 tool under test.
28 3. If the environment member variable exists, its write() method will be
29 invoked.
30 4. All expected_* member variables will be inspected and all placeholders in
31 them will be expanded by calling instantiate_for_expectation() on those
32 placeholders. After placeholder expansion, if the expected_* variable is
33 a list, its element will be joined together with '' to form a single
34 string. These expected_* variables are to be used by the check_*() methods.
35 5. The spirv-* tool will be run with the arguments supplied in spirv_args.
36 6. All check_*() member methods will be called by supplying a TestStatus as
37 argument. Each check_*() method is expected to return a (Success, Message)
38 pair where Success is a boolean indicating success and Message is an error
39 message.
40 7. If any check_*() method fails, the error message is output and the
41 current test case fails.
42
43 If --leave-output was not specified, all temporary files and directories will
44 be deleted.
45 """
46
47 import argparse
48 import fnmatch
49 import inspect
50 import os
51 import shutil
52 import subprocess
53 import sys
54 import tempfile
55 from collections import defaultdict
56 from placeholder import PlaceHolder
57
58 EXPECTED_BEHAVIOR_PREFIX = 'expected_'
59 VALIDATE_METHOD_PREFIX = 'check_'
60
61
62 def get_all_variables(instance):
63 """Returns the names of all the variables in instance."""
64 return [v for v in dir(instance) if not callable(getattr(instance, v))]
65
66
67 def get_all_methods(instance):
68 """Returns the names of all methods in instance."""
69 return [m for m in dir(instance) if callable(getattr(instance, m))]
70
71
72 def get_all_superclasses(cls):
73 """Returns all superclasses of a given class. Omits root 'object' superclass.
74
75 Returns:
76 A list of superclasses of the given class. The order guarantees that
77 * A Base class precedes its derived classes, e.g., for "class B(A)", it
78 will be [..., A, B, ...].
79 * When there are multiple base classes, base classes declared first
80 precede those declared later, e.g., for "class C(A, B), it will be
81 [..., A, B, C, ...]
82 """
83 classes = []
84 for superclass in cls.__bases__:
85 for c in get_all_superclasses(superclass):
86 if c is not object and c not in classes:
87 classes.append(c)
88 for superclass in cls.__bases__:
89 if superclass is not object and superclass not in classes:
90 classes.append(superclass)
91
92 return classes
93
94
95 def get_all_test_methods(test_class):
96 """Gets all validation methods.
97
98 Returns:
99 A list of validation methods. The order guarantees that
100 * A method defined in superclass precedes one defined in subclass,
101 e.g., for "class A(B)", methods defined in B precedes those defined
102 in A.
103 * If a subclass has more than one superclass, e.g., "class C(A, B)",
104 then methods defined in A precedes those defined in B.
105 """
106 classes = get_all_superclasses(test_class)
107 classes.append(test_class)
108 all_tests = [
109 m for c in classes for m in get_all_methods(c)
110 if m.startswith(VALIDATE_METHOD_PREFIX)
111 ]
112 unique_tests = []
113 for t in all_tests:
114 if t not in unique_tests:
115 unique_tests.append(t)
116 return unique_tests
117
118
119 class SpirvTest:
120 """Base class for spirv test cases.
121
122 Subclasses define test cases' facts (shader source code, spirv command,
123 result validation), which will be used by the TestCase class for running
124 tests. Subclasses should define spirv_args (specifying spirv_tool command
125 arguments), and at least one check_*() method (for result validation) for
126 a full-fledged test case. All check_*() methods should take a TestStatus
127 parameter and return a (Success, Message) pair, in which Success is a
128 boolean indicating success and Message is an error message. The test passes
129 iff all check_*() methods returns true.
130
131 Often, a test case class will delegate the check_* behaviors by inheriting
132 from other classes.
133 """
134
135 def name(self):
136 return self.__class__.__name__
137
138
139 class TestStatus:
140 """A struct for holding run status of a test case."""
141
142 def __init__(self, test_manager, returncode, stdout, stderr, directory,
143 inputs, input_filenames):
144 self.test_manager = test_manager
145 self.returncode = returncode
146 # Some of our MacOS bots still run Python 2, so need to be backwards
147 # compatible here.
148 if type(stdout) is not str:
149 if sys.version_info[0] == 2:
150 self.stdout = stdout.decode('utf-8')
151 elif sys.version_info[0] == 3:
152 self.stdout = str(stdout, encoding='utf-8') if stdout is not None else stdout
153 else:
154 raise Exception('Unable to determine if running Python 2 or 3 from {}'.format(sys.version_info))
155 else:
156 self.stdout = stdout
157
158 if type(stderr) is not str:
159 if sys.version_info[0] == 2:
160 self.stderr = stderr.decode('utf-8')
161 elif sys.version_info[0] == 3:
162 self.stderr = str(stderr, encoding='utf-8') if stderr is not None else stderr
163 else:
164 raise Exception('Unable to determine if running Python 2 or 3 from {}'.format(sys.version_info))
165 else:
166 self.stderr = stderr
167
168 # temporary directory where the test runs
169 self.directory = directory
170 # List of inputs, as PlaceHolder objects.
171 self.inputs = inputs
172 # the names of input shader files (potentially including paths)
173 self.input_filenames = input_filenames
174
175
176 class SpirvTestException(Exception):
177 """SpirvTest exception class."""
178 pass
179
180
181 def inside_spirv_testsuite(testsuite_name):
182 """Decorator for subclasses of SpirvTest.
183
184 This decorator checks that a class meets the requirements (see below)
185 for a test case class, and then puts the class in a certain testsuite.
186 * The class needs to be a subclass of SpirvTest.
187 * The class needs to have spirv_args defined as a list.
188 * The class needs to define at least one check_*() methods.
189 * All expected_* variables required by check_*() methods can only be
190 of bool, str, or list type.
191 * Python runtime will throw an exception if the expected_* member
192 attributes required by check_*() methods are missing.
193 """
194
195 def actual_decorator(cls):
196 if not inspect.isclass(cls):
197 raise SpirvTestException('Test case should be a class')
198 if not issubclass(cls, SpirvTest):
199 raise SpirvTestException(
200 'All test cases should be subclasses of SpirvTest')
201 if 'spirv_args' not in get_all_variables(cls):
202 raise SpirvTestException('No spirv_args found in the test case')
203 if not isinstance(cls.spirv_args, list):
204 raise SpirvTestException('spirv_args needs to be a list')
205 if not any(
206 [m.startswith(VALIDATE_METHOD_PREFIX) for m in get_all_methods(cls)]):
207 raise SpirvTestException('No check_*() methods found in the test case')
208 if not all(
209 [isinstance(v, (bool, str, list)) for v in get_all_variables(cls)]):
210 raise SpirvTestException(
211 'expected_* variables are only allowed to be bool, str, or '
212 'list type.')
213 cls.parent_testsuite = testsuite_name
214 return cls
215
216 return actual_decorator
217
218
219 class TestManager:
220 """Manages and runs a set of tests."""
221
222 def __init__(self, executable_path, assembler_path, disassembler_path):
223 self.executable_path = executable_path
224 self.assembler_path = assembler_path
225 self.disassembler_path = disassembler_path
226 self.num_successes = 0
227 self.num_failures = 0
228 self.num_tests = 0
229 self.leave_output = False
230 self.tests = defaultdict(list)
231
232 def notify_result(self, test_case, success, message):
233 """Call this to notify the manager of the results of a test run."""
234 self.num_successes += 1 if success else 0
235 self.num_failures += 0 if success else 1
236 counter_string = str(self.num_successes + self.num_failures) + '/' + str(
237 self.num_tests)
238 print('%-10s %-40s ' % (counter_string, test_case.test.name()) +
239 ('Passed' if success else '-Failed-'))
240 if not success:
241 print(' '.join(test_case.command))
242 print(message)
243
244 def add_test(self, testsuite, test):
245 """Add this to the current list of test cases."""
246 self.tests[testsuite].append(TestCase(test, self))
247 self.num_tests += 1
248
249 def run_tests(self):
250 for suite in self.tests:
251 print('SPIRV tool test suite: "{suite}"'.format(suite=suite))
252 for x in self.tests[suite]:
253 x.runTest()
254
255
256 class TestCase:
257 """A single test case that runs in its own directory."""
258
259 def __init__(self, test, test_manager):
260 self.test = test
261 self.test_manager = test_manager
262 self.inputs = [] # inputs, as PlaceHolder objects.
263 self.file_shaders = [] # filenames of shader files.
264 self.stdin_shader = None # text to be passed to spirv_tool as stdin
265
266 def setUp(self):
267 """Creates environment and instantiates placeholders for the test case."""
268
269 self.directory = tempfile.mkdtemp(dir=os.getcwd())
270 spirv_args = self.test.spirv_args
271 # Instantiate placeholders in spirv_args
272 self.test.spirv_args = [
273 arg.instantiate_for_spirv_args(self)
274 if isinstance(arg, PlaceHolder) else arg for arg in self.test.spirv_args
275 ]
276 # Get all shader files' names
277 self.inputs = [arg for arg in spirv_args if isinstance(arg, PlaceHolder)]
278 self.file_shaders = [arg.filename for arg in self.inputs]
279
280 if 'environment' in get_all_variables(self.test):
281 self.test.environment.write(self.directory)
282
283 expectations = [
284 v for v in get_all_variables(self.test)
285 if v.startswith(EXPECTED_BEHAVIOR_PREFIX)
286 ]
287 # Instantiate placeholders in expectations
288 for expectation_name in expectations:
289 expectation = getattr(self.test, expectation_name)
290 if isinstance(expectation, list):
291 expanded_expections = [
292 element.instantiate_for_expectation(self)
293 if isinstance(element, PlaceHolder) else element
294 for element in expectation
295 ]
296 setattr(self.test, expectation_name, expanded_expections)
297 elif isinstance(expectation, PlaceHolder):
298 setattr(self.test, expectation_name,
299 expectation.instantiate_for_expectation(self))
300
301 def tearDown(self):
302 """Removes the directory if we were not instructed to do otherwise."""
303 if not self.test_manager.leave_output:
304 shutil.rmtree(self.directory)
305
306 def runTest(self):
307 """Sets up and runs a test, reports any failures and then cleans up."""
308 self.setUp()
309 success = False
310 message = ''
311 try:
312 self.command = [self.test_manager.executable_path]
313 self.command.extend(self.test.spirv_args)
314
315 process = subprocess.Popen(
316 args=self.command,
317 stdin=subprocess.PIPE,
318 stdout=subprocess.PIPE,
319 stderr=subprocess.PIPE,
320 cwd=self.directory)
321 output = process.communicate(self.stdin_shader)
322 test_status = TestStatus(self.test_manager, process.returncode, output[0],
323 output[1], self.directory, self.inputs,
324 self.file_shaders)
325 run_results = [
326 getattr(self.test, test_method)(test_status)
327 for test_method in get_all_test_methods(self.test.__class__)
328 ]
329 success, message = zip(*run_results)
330 success = all(success)
331 message = '\n'.join(message)
332 except Exception as e:
333 success = False
334 message = str(e)
335 self.test_manager.notify_result(
336 self, success,
337 message + '\nSTDOUT:\n%s\nSTDERR:\n%s' % (output[0], output[1]))
338 self.tearDown()
339
340
341 def main():
342 parser = argparse.ArgumentParser()
343 parser.add_argument(
344 'spirv_tool',
345 metavar='path/to/spirv_tool',
346 type=str,
347 nargs=1,
348 help='Path to the spirv-* tool under test')
349 parser.add_argument(
350 'spirv_as',
351 metavar='path/to/spirv-as',
352 type=str,
353 nargs=1,
354 help='Path to spirv-as')
355 parser.add_argument(
356 'spirv_dis',
357 metavar='path/to/spirv-dis',
358 type=str,
359 nargs=1,
360 help='Path to spirv-dis')
361 parser.add_argument(
362 '--leave-output',
363 action='store_const',
364 const=1,
365 help='Do not clean up temporary directories')
366 parser.add_argument(
367 '--test-dir', nargs=1, help='Directory to gather the tests from')
368 args = parser.parse_args()
369 default_path = sys.path
370 root_dir = os.getcwd()
371 if args.test_dir:
372 root_dir = args.test_dir[0]
373 manager = TestManager(args.spirv_tool[0], args.spirv_as[0], args.spirv_dis[0])
374 if args.leave_output:
375 manager.leave_output = True
376 for root, _, filenames in os.walk(root_dir):
377 for filename in fnmatch.filter(filenames, '*.py'):
378 if filename.endswith('nosetest.py'):
379 # Skip nose tests, which are for testing functions of
380 # the test framework.
381 continue
382 sys.path = default_path
383 sys.path.append(root)
384 mod = __import__(os.path.splitext(filename)[0])
385 for _, obj, in inspect.getmembers(mod):
386 if inspect.isclass(obj) and hasattr(obj, 'parent_testsuite'):
387 manager.add_test(obj.parent_testsuite, obj())
388 manager.run_tests()
389 if manager.num_failures > 0:
390 sys.exit(-1)
391
392
393 if __name__ == '__main__':
394 main()