| 1 |
# Copyright (c) 2018 Google LLC |
| 2 |
# |
| 3 |
# Licensed under the Apache License, Version 2.0 (the "License"); |
| 4 |
# you may not use this file except in compliance with the License. |
| 5 |
# You may obtain a copy of the License at |
| 6 |
# |
| 7 |
# http://www.apache.org/licenses/LICENSE-2.0 |
| 8 |
# |
| 9 |
# Unless required by applicable law or agreed to in writing, software |
| 10 |
# distributed under the License is distributed on an "AS IS" BASIS, |
| 11 |
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| 12 |
# See the License for the specific language governing permissions and |
| 13 |
# limitations under the License. |
| 14 |
"""Manages and runs tests from the current working directory. |
| 15 |
|
| 16 |
This will traverse the current working directory and look for python files that |
| 17 |
contain subclasses of SpirvTest. |
| 18 |
|
| 19 |
If a class has an @inside_spirv_testsuite decorator, an instance of that |
| 20 |
class will be created and serve as a test case in that testsuite. The test |
| 21 |
case is then run by the following steps: |
| 22 |
|
| 23 |
1. A temporary directory will be created. |
| 24 |
2. The spirv_args member variable will be inspected and all placeholders in it |
| 25 |
will be expanded by calling instantiate_for_spirv_args() on placeholders. |
| 26 |
The transformed list elements are then supplied as arguments to the spirv-* |
| 27 |
tool under test. |
| 28 |
3. If the environment member variable exists, its write() method will be |
| 29 |
invoked. |
| 30 |
4. All expected_* member variables will be inspected and all placeholders in |
| 31 |
them will be expanded by calling instantiate_for_expectation() on those |
| 32 |
placeholders. After placeholder expansion, if the expected_* variable is |
| 33 |
a list, its element will be joined together with '' to form a single |
| 34 |
string. These expected_* variables are to be used by the check_*() methods. |
| 35 |
5. The spirv-* tool will be run with the arguments supplied in spirv_args. |
| 36 |
6. All check_*() member methods will be called by supplying a TestStatus as |
| 37 |
argument. Each check_*() method is expected to return a (Success, Message) |
| 38 |
pair where Success is a boolean indicating success and Message is an error |
| 39 |
message. |
| 40 |
7. If any check_*() method fails, the error message is output and the |
| 41 |
current test case fails. |
| 42 |
|
| 43 |
If --leave-output was not specified, all temporary files and directories will |
| 44 |
be deleted. |
| 45 |
""" |
| 46 |
|
| 47 |
import argparse |
| 48 |
import fnmatch |
| 49 |
import inspect |
| 50 |
import os |
| 51 |
import shutil |
| 52 |
import subprocess |
| 53 |
import sys |
| 54 |
import tempfile |
| 55 |
from collections import defaultdict |
| 56 |
from placeholder import PlaceHolder |
| 57 |
|
| 58 |
EXPECTED_BEHAVIOR_PREFIX = 'expected_' |
| 59 |
VALIDATE_METHOD_PREFIX = 'check_' |
| 60 |
|
| 61 |
|
| 62 |
def get_all_variables(instance): |
| 63 |
"""Returns the names of all the variables in instance.""" |
| 64 |
return [v for v in dir(instance) if not callable(getattr(instance, v))] |
| 65 |
|
| 66 |
|
| 67 |
def get_all_methods(instance): |
| 68 |
"""Returns the names of all methods in instance.""" |
| 69 |
return [m for m in dir(instance) if callable(getattr(instance, m))] |
| 70 |
|
| 71 |
|
| 72 |
def get_all_superclasses(cls): |
| 73 |
"""Returns all superclasses of a given class. Omits root 'object' superclass. |
| 74 |
|
| 75 |
Returns: |
| 76 |
A list of superclasses of the given class. The order guarantees that |
| 77 |
* A Base class precedes its derived classes, e.g., for "class B(A)", it |
| 78 |
will be [..., A, B, ...]. |
| 79 |
* When there are multiple base classes, base classes declared first |
| 80 |
precede those declared later, e.g., for "class C(A, B), it will be |
| 81 |
[..., A, B, C, ...] |
| 82 |
""" |
| 83 |
classes = [] |
| 84 |
for superclass in cls.__bases__: |
| 85 |
for c in get_all_superclasses(superclass): |
| 86 |
if c is not object and c not in classes: |
| 87 |
classes.append(c) |
| 88 |
for superclass in cls.__bases__: |
| 89 |
if superclass is not object and superclass not in classes: |
| 90 |
classes.append(superclass) |
| 91 |
|
| 92 |
return classes |
| 93 |
|
| 94 |
|
| 95 |
def get_all_test_methods(test_class): |
| 96 |
"""Gets all validation methods. |
| 97 |
|
| 98 |
Returns: |
| 99 |
A list of validation methods. The order guarantees that |
| 100 |
* A method defined in superclass precedes one defined in subclass, |
| 101 |
e.g., for "class A(B)", methods defined in B precedes those defined |
| 102 |
in A. |
| 103 |
* If a subclass has more than one superclass, e.g., "class C(A, B)", |
| 104 |
then methods defined in A precedes those defined in B. |
| 105 |
""" |
| 106 |
classes = get_all_superclasses(test_class) |
| 107 |
classes.append(test_class) |
| 108 |
all_tests = [ |
| 109 |
m for c in classes for m in get_all_methods(c) |
| 110 |
if m.startswith(VALIDATE_METHOD_PREFIX) |
| 111 |
] |
| 112 |
unique_tests = [] |
| 113 |
for t in all_tests: |
| 114 |
if t not in unique_tests: |
| 115 |
unique_tests.append(t) |
| 116 |
return unique_tests |
| 117 |
|
| 118 |
|
| 119 |
class SpirvTest: |
| 120 |
"""Base class for spirv test cases. |
| 121 |
|
| 122 |
Subclasses define test cases' facts (shader source code, spirv command, |
| 123 |
result validation), which will be used by the TestCase class for running |
| 124 |
tests. Subclasses should define spirv_args (specifying spirv_tool command |
| 125 |
arguments), and at least one check_*() method (for result validation) for |
| 126 |
a full-fledged test case. All check_*() methods should take a TestStatus |
| 127 |
parameter and return a (Success, Message) pair, in which Success is a |
| 128 |
boolean indicating success and Message is an error message. The test passes |
| 129 |
iff all check_*() methods returns true. |
| 130 |
|
| 131 |
Often, a test case class will delegate the check_* behaviors by inheriting |
| 132 |
from other classes. |
| 133 |
""" |
| 134 |
|
| 135 |
def name(self): |
| 136 |
return self.__class__.__name__ |
| 137 |
|
| 138 |
|
| 139 |
class TestStatus: |
| 140 |
"""A struct for holding run status of a test case.""" |
| 141 |
|
| 142 |
def __init__(self, test_manager, returncode, stdout, stderr, directory, |
| 143 |
inputs, input_filenames): |
| 144 |
self.test_manager = test_manager |
| 145 |
self.returncode = returncode |
| 146 |
# Some of our MacOS bots still run Python 2, so need to be backwards |
| 147 |
# compatible here. |
| 148 |
if type(stdout) is not str: |
| 149 |
if sys.version_info[0] == 2: |
| 150 |
self.stdout = stdout.decode('utf-8') |
| 151 |
elif sys.version_info[0] == 3: |
| 152 |
self.stdout = str(stdout, encoding='utf-8') if stdout is not None else stdout |
| 153 |
else: |
| 154 |
raise Exception('Unable to determine if running Python 2 or 3 from {}'.format(sys.version_info)) |
| 155 |
else: |
| 156 |
self.stdout = stdout |
| 157 |
|
| 158 |
if type(stderr) is not str: |
| 159 |
if sys.version_info[0] == 2: |
| 160 |
self.stderr = stderr.decode('utf-8') |
| 161 |
elif sys.version_info[0] == 3: |
| 162 |
self.stderr = str(stderr, encoding='utf-8') if stderr is not None else stderr |
| 163 |
else: |
| 164 |
raise Exception('Unable to determine if running Python 2 or 3 from {}'.format(sys.version_info)) |
| 165 |
else: |
| 166 |
self.stderr = stderr |
| 167 |
|
| 168 |
# temporary directory where the test runs |
| 169 |
self.directory = directory |
| 170 |
# List of inputs, as PlaceHolder objects. |
| 171 |
self.inputs = inputs |
| 172 |
# the names of input shader files (potentially including paths) |
| 173 |
self.input_filenames = input_filenames |
| 174 |
|
| 175 |
|
| 176 |
class SpirvTestException(Exception): |
| 177 |
"""SpirvTest exception class.""" |
| 178 |
pass |
| 179 |
|
| 180 |
|
| 181 |
def inside_spirv_testsuite(testsuite_name): |
| 182 |
"""Decorator for subclasses of SpirvTest. |
| 183 |
|
| 184 |
This decorator checks that a class meets the requirements (see below) |
| 185 |
for a test case class, and then puts the class in a certain testsuite. |
| 186 |
* The class needs to be a subclass of SpirvTest. |
| 187 |
* The class needs to have spirv_args defined as a list. |
| 188 |
* The class needs to define at least one check_*() methods. |
| 189 |
* All expected_* variables required by check_*() methods can only be |
| 190 |
of bool, str, or list type. |
| 191 |
* Python runtime will throw an exception if the expected_* member |
| 192 |
attributes required by check_*() methods are missing. |
| 193 |
""" |
| 194 |
|
| 195 |
def actual_decorator(cls): |
| 196 |
if not inspect.isclass(cls): |
| 197 |
raise SpirvTestException('Test case should be a class') |
| 198 |
if not issubclass(cls, SpirvTest): |
| 199 |
raise SpirvTestException( |
| 200 |
'All test cases should be subclasses of SpirvTest') |
| 201 |
if 'spirv_args' not in get_all_variables(cls): |
| 202 |
raise SpirvTestException('No spirv_args found in the test case') |
| 203 |
if not isinstance(cls.spirv_args, list): |
| 204 |
raise SpirvTestException('spirv_args needs to be a list') |
| 205 |
if not any( |
| 206 |
[m.startswith(VALIDATE_METHOD_PREFIX) for m in get_all_methods(cls)]): |
| 207 |
raise SpirvTestException('No check_*() methods found in the test case') |
| 208 |
if not all( |
| 209 |
[isinstance(v, (bool, str, list)) for v in get_all_variables(cls)]): |
| 210 |
raise SpirvTestException( |
| 211 |
'expected_* variables are only allowed to be bool, str, or ' |
| 212 |
'list type.') |
| 213 |
cls.parent_testsuite = testsuite_name |
| 214 |
return cls |
| 215 |
|
| 216 |
return actual_decorator |
| 217 |
|
| 218 |
|
| 219 |
class TestManager: |
| 220 |
"""Manages and runs a set of tests.""" |
| 221 |
|
| 222 |
def __init__(self, executable_path, assembler_path, disassembler_path): |
| 223 |
self.executable_path = executable_path |
| 224 |
self.assembler_path = assembler_path |
| 225 |
self.disassembler_path = disassembler_path |
| 226 |
self.num_successes = 0 |
| 227 |
self.num_failures = 0 |
| 228 |
self.num_tests = 0 |
| 229 |
self.leave_output = False |
| 230 |
self.tests = defaultdict(list) |
| 231 |
|
| 232 |
def notify_result(self, test_case, success, message): |
| 233 |
"""Call this to notify the manager of the results of a test run.""" |
| 234 |
self.num_successes += 1 if success else 0 |
| 235 |
self.num_failures += 0 if success else 1 |
| 236 |
counter_string = str(self.num_successes + self.num_failures) + '/' + str( |
| 237 |
self.num_tests) |
| 238 |
print('%-10s %-40s ' % (counter_string, test_case.test.name()) + |
| 239 |
('Passed' if success else '-Failed-')) |
| 240 |
if not success: |
| 241 |
print(' '.join(test_case.command)) |
| 242 |
print(message) |
| 243 |
|
| 244 |
def add_test(self, testsuite, test): |
| 245 |
"""Add this to the current list of test cases.""" |
| 246 |
self.tests[testsuite].append(TestCase(test, self)) |
| 247 |
self.num_tests += 1 |
| 248 |
|
| 249 |
def run_tests(self): |
| 250 |
for suite in self.tests: |
| 251 |
print('SPIRV tool test suite: "{suite}"'.format(suite=suite)) |
| 252 |
for x in self.tests[suite]: |
| 253 |
x.runTest() |
| 254 |
|
| 255 |
|
| 256 |
class TestCase: |
| 257 |
"""A single test case that runs in its own directory.""" |
| 258 |
|
| 259 |
def __init__(self, test, test_manager): |
| 260 |
self.test = test |
| 261 |
self.test_manager = test_manager |
| 262 |
self.inputs = [] # inputs, as PlaceHolder objects. |
| 263 |
self.file_shaders = [] # filenames of shader files. |
| 264 |
self.stdin_shader = None # text to be passed to spirv_tool as stdin |
| 265 |
|
| 266 |
def setUp(self): |
| 267 |
"""Creates environment and instantiates placeholders for the test case.""" |
| 268 |
|
| 269 |
self.directory = tempfile.mkdtemp(dir=os.getcwd()) |
| 270 |
spirv_args = self.test.spirv_args |
| 271 |
# Instantiate placeholders in spirv_args |
| 272 |
self.test.spirv_args = [ |
| 273 |
arg.instantiate_for_spirv_args(self) |
| 274 |
if isinstance(arg, PlaceHolder) else arg for arg in self.test.spirv_args |
| 275 |
] |
| 276 |
# Get all shader files' names |
| 277 |
self.inputs = [arg for arg in spirv_args if isinstance(arg, PlaceHolder)] |
| 278 |
self.file_shaders = [arg.filename for arg in self.inputs] |
| 279 |
|
| 280 |
if 'environment' in get_all_variables(self.test): |
| 281 |
self.test.environment.write(self.directory) |
| 282 |
|
| 283 |
expectations = [ |
| 284 |
v for v in get_all_variables(self.test) |
| 285 |
if v.startswith(EXPECTED_BEHAVIOR_PREFIX) |
| 286 |
] |
| 287 |
# Instantiate placeholders in expectations |
| 288 |
for expectation_name in expectations: |
| 289 |
expectation = getattr(self.test, expectation_name) |
| 290 |
if isinstance(expectation, list): |
| 291 |
expanded_expections = [ |
| 292 |
element.instantiate_for_expectation(self) |
| 293 |
if isinstance(element, PlaceHolder) else element |
| 294 |
for element in expectation |
| 295 |
] |
| 296 |
setattr(self.test, expectation_name, expanded_expections) |
| 297 |
elif isinstance(expectation, PlaceHolder): |
| 298 |
setattr(self.test, expectation_name, |
| 299 |
expectation.instantiate_for_expectation(self)) |
| 300 |
|
| 301 |
def tearDown(self): |
| 302 |
"""Removes the directory if we were not instructed to do otherwise.""" |
| 303 |
if not self.test_manager.leave_output: |
| 304 |
shutil.rmtree(self.directory) |
| 305 |
|
| 306 |
def runTest(self): |
| 307 |
"""Sets up and runs a test, reports any failures and then cleans up.""" |
| 308 |
self.setUp() |
| 309 |
success = False |
| 310 |
message = '' |
| 311 |
try: |
| 312 |
self.command = [self.test_manager.executable_path] |
| 313 |
self.command.extend(self.test.spirv_args) |
| 314 |
|
| 315 |
process = subprocess.Popen( |
| 316 |
args=self.command, |
| 317 |
stdin=subprocess.PIPE, |
| 318 |
stdout=subprocess.PIPE, |
| 319 |
stderr=subprocess.PIPE, |
| 320 |
cwd=self.directory) |
| 321 |
output = process.communicate(self.stdin_shader) |
| 322 |
test_status = TestStatus(self.test_manager, process.returncode, output[0], |
| 323 |
output[1], self.directory, self.inputs, |
| 324 |
self.file_shaders) |
| 325 |
run_results = [ |
| 326 |
getattr(self.test, test_method)(test_status) |
| 327 |
for test_method in get_all_test_methods(self.test.__class__) |
| 328 |
] |
| 329 |
success, message = zip(*run_results) |
| 330 |
success = all(success) |
| 331 |
message = '\n'.join(message) |
| 332 |
except Exception as e: |
| 333 |
success = False |
| 334 |
message = str(e) |
| 335 |
self.test_manager.notify_result( |
| 336 |
self, success, |
| 337 |
message + '\nSTDOUT:\n%s\nSTDERR:\n%s' % (output[0], output[1])) |
| 338 |
self.tearDown() |
| 339 |
|
| 340 |
|
| 341 |
def main(): |
| 342 |
parser = argparse.ArgumentParser() |
| 343 |
parser.add_argument( |
| 344 |
'spirv_tool', |
| 345 |
metavar='path/to/spirv_tool', |
| 346 |
type=str, |
| 347 |
nargs=1, |
| 348 |
help='Path to the spirv-* tool under test') |
| 349 |
parser.add_argument( |
| 350 |
'spirv_as', |
| 351 |
metavar='path/to/spirv-as', |
| 352 |
type=str, |
| 353 |
nargs=1, |
| 354 |
help='Path to spirv-as') |
| 355 |
parser.add_argument( |
| 356 |
'spirv_dis', |
| 357 |
metavar='path/to/spirv-dis', |
| 358 |
type=str, |
| 359 |
nargs=1, |
| 360 |
help='Path to spirv-dis') |
| 361 |
parser.add_argument( |
| 362 |
'--leave-output', |
| 363 |
action='store_const', |
| 364 |
const=1, |
| 365 |
help='Do not clean up temporary directories') |
| 366 |
parser.add_argument( |
| 367 |
'--test-dir', nargs=1, help='Directory to gather the tests from') |
| 368 |
args = parser.parse_args() |
| 369 |
default_path = sys.path |
| 370 |
root_dir = os.getcwd() |
| 371 |
if args.test_dir: |
| 372 |
root_dir = args.test_dir[0] |
| 373 |
manager = TestManager(args.spirv_tool[0], args.spirv_as[0], args.spirv_dis[0]) |
| 374 |
if args.leave_output: |
| 375 |
manager.leave_output = True |
| 376 |
for root, _, filenames in os.walk(root_dir): |
| 377 |
for filename in fnmatch.filter(filenames, '*.py'): |
| 378 |
if filename.endswith('nosetest.py'): |
| 379 |
# Skip nose tests, which are for testing functions of |
| 380 |
# the test framework. |
| 381 |
continue |
| 382 |
sys.path = default_path |
| 383 |
sys.path.append(root) |
| 384 |
mod = __import__(os.path.splitext(filename)[0]) |
| 385 |
for _, obj, in inspect.getmembers(mod): |
| 386 |
if inspect.isclass(obj) and hasattr(obj, 'parent_testsuite'): |
| 387 |
manager.add_test(obj.parent_testsuite, obj()) |
| 388 |
manager.run_tests() |
| 389 |
if manager.num_failures > 0: |
| 390 |
sys.exit(-1) |
| 391 |
|
| 392 |
|
| 393 |
if __name__ == '__main__': |
| 394 |
main() |