| 1 |
#!/usr/bin/env python |
| 2 |
|
| 3 |
""" |
| 4 |
Static Analyzer qualification infrastructure. |
| 5 |
|
| 6 |
The goal is to test the analyzer against different projects, |
| 7 |
check for failures, compare results, and measure performance. |
| 8 |
|
| 9 |
Repository Directory will contain sources of the projects as well as the |
| 10 |
information on how to build them and the expected output. |
| 11 |
Repository Directory structure: |
| 12 |
- ProjectMap file |
| 13 |
- Historical Performance Data |
| 14 |
- Project Dir1 |
| 15 |
- ReferenceOutput |
| 16 |
- Project Dir2 |
| 17 |
- ReferenceOutput |
| 18 |
.. |
| 19 |
Note that the build tree must be inside the project dir. |
| 20 |
|
| 21 |
To test the build of the analyzer one would: |
| 22 |
- Copy over a copy of the Repository Directory. (TODO: Prefer to ensure that |
| 23 |
the build directory does not pollute the repository to min network |
| 24 |
traffic). |
| 25 |
- Build all projects, until error. Produce logs to report errors. |
| 26 |
- Compare results. |
| 27 |
|
| 28 |
The files which should be kept around for failure investigations: |
| 29 |
RepositoryCopy/Project DirI/ScanBuildResults |
| 30 |
RepositoryCopy/Project DirI/run_static_analyzer.log |
| 31 |
|
| 32 |
Assumptions (TODO: shouldn't need to assume these.): |
| 33 |
The script is being run from the Repository Directory. |
| 34 |
The compiler for scan-build and scan-build are in the PATH. |
| 35 |
export PATH=/Users/zaks/workspace/c2llvm/build/Release+Asserts/bin:$PATH |
| 36 |
|
| 37 |
For more logging, set the env variables: |
| 38 |
zaks:TI zaks$ export CCC_ANALYZER_LOG=1 |
| 39 |
zaks:TI zaks$ export CCC_ANALYZER_VERBOSE=1 |
| 40 |
|
| 41 |
The list of checkers tested are hardcoded in the Checkers variable. |
| 42 |
For testing additional checkers, use the SA_ADDITIONAL_CHECKERS environment |
| 43 |
variable. It should contain a comma separated list. |
| 44 |
""" |
| 45 |
import CmpRuns |
| 46 |
import SATestUtils as utils |
| 47 |
from ProjectMap import DownloadType, ProjectInfo |
| 48 |
|
| 49 |
import glob |
| 50 |
import logging |
| 51 |
import math |
| 52 |
import multiprocessing |
| 53 |
import os |
| 54 |
import plistlib |
| 55 |
import shutil |
| 56 |
import sys |
| 57 |
import threading |
| 58 |
import time |
| 59 |
import zipfile |
| 60 |
|
| 61 |
from queue import Queue |
| 62 |
# mypy has problems finding InvalidFileException in the module |
| 63 |
# and this is we can shush that false positive |
| 64 |
from plistlib import InvalidFileException # type:ignore |
| 65 |
from subprocess import CalledProcessError, check_call |
| 66 |
from typing import Dict, IO, List, NamedTuple, Optional, TYPE_CHECKING, Tuple |
| 67 |
|
| 68 |
|
| 69 |
############################################################################### |
| 70 |
# Helper functions. |
| 71 |
############################################################################### |
| 72 |
|
| 73 |
class StreamToLogger: |
| 74 |
def __init__(self, logger: logging.Logger, |
| 75 |
log_level: int = logging.INFO): |
| 76 |
self.logger = logger |
| 77 |
self.log_level = log_level |
| 78 |
|
| 79 |
def write(self, message: str): |
| 80 |
# Rstrip in order not to write an extra newline. |
| 81 |
self.logger.log(self.log_level, message.rstrip()) |
| 82 |
|
| 83 |
def flush(self): |
| 84 |
pass |
| 85 |
|
| 86 |
def fileno(self) -> int: |
| 87 |
return 0 |
| 88 |
|
| 89 |
|
| 90 |
LOCAL = threading.local() |
| 91 |
|
| 92 |
|
| 93 |
def init_logger(name: str): |
| 94 |
# TODO: use debug levels for VERBOSE messages |
| 95 |
logger = logging.getLogger(name) |
| 96 |
logger.setLevel(logging.DEBUG) |
| 97 |
LOCAL.stdout = StreamToLogger(logger, logging.INFO) |
| 98 |
LOCAL.stderr = StreamToLogger(logger, logging.ERROR) |
| 99 |
|
| 100 |
|
| 101 |
init_logger("main") |
| 102 |
|
| 103 |
|
| 104 |
def stderr(message: str): |
| 105 |
LOCAL.stderr.write(message) |
| 106 |
|
| 107 |
|
| 108 |
def stdout(message: str): |
| 109 |
LOCAL.stdout.write(message) |
| 110 |
|
| 111 |
|
| 112 |
logging.basicConfig( |
| 113 |
format='%(asctime)s:%(levelname)s:%(name)s: %(message)s') |
| 114 |
|
| 115 |
|
| 116 |
############################################################################### |
| 117 |
# Configuration setup. |
| 118 |
############################################################################### |
| 119 |
|
| 120 |
|
| 121 |
# Find Clang for static analysis. |
| 122 |
if 'CC' in os.environ: |
| 123 |
cc_candidate: Optional[str] = os.environ['CC'] |
| 124 |
else: |
| 125 |
cc_candidate = utils.which("clang", os.environ['PATH']) |
| 126 |
if not cc_candidate: |
| 127 |
stderr("Error: cannot find 'clang' in PATH") |
| 128 |
sys.exit(1) |
| 129 |
|
| 130 |
CLANG = cc_candidate |
| 131 |
|
| 132 |
# Number of jobs. |
| 133 |
MAX_JOBS = int(math.ceil(multiprocessing.cpu_count() * 0.75)) |
| 134 |
|
| 135 |
# Names of the project specific scripts. |
| 136 |
# The script that downloads the project. |
| 137 |
DOWNLOAD_SCRIPT = "download_project.sh" |
| 138 |
# The script that needs to be executed before the build can start. |
| 139 |
CLEANUP_SCRIPT = "cleanup_run_static_analyzer.sh" |
| 140 |
# This is a file containing commands for scan-build. |
| 141 |
BUILD_SCRIPT = "run_static_analyzer.cmd" |
| 142 |
|
| 143 |
# A comment in a build script which disables wrapping. |
| 144 |
NO_PREFIX_CMD = "#NOPREFIX" |
| 145 |
|
| 146 |
# The log file name. |
| 147 |
LOG_DIR_NAME = "Logs" |
| 148 |
BUILD_LOG_NAME = "run_static_analyzer.log" |
| 149 |
# Summary file - contains the summary of the failures. Ex: This info can be be |
| 150 |
# displayed when buildbot detects a build failure. |
| 151 |
NUM_OF_FAILURES_IN_SUMMARY = 10 |
| 152 |
|
| 153 |
# The scan-build result directory. |
| 154 |
OUTPUT_DIR_NAME = "ScanBuildResults" |
| 155 |
REF_PREFIX = "Ref" |
| 156 |
|
| 157 |
# The name of the directory storing the cached project source. If this |
| 158 |
# directory does not exist, the download script will be executed. |
| 159 |
# That script should create the "CachedSource" directory and download the |
| 160 |
# project source into it. |
| 161 |
CACHED_SOURCE_DIR_NAME = "CachedSource" |
| 162 |
|
| 163 |
# The name of the directory containing the source code that will be analyzed. |
| 164 |
# Each time a project is analyzed, a fresh copy of its CachedSource directory |
| 165 |
# will be copied to the PatchedSource directory and then the local patches |
| 166 |
# in PATCHFILE_NAME will be applied (if PATCHFILE_NAME exists). |
| 167 |
PATCHED_SOURCE_DIR_NAME = "PatchedSource" |
| 168 |
|
| 169 |
# The name of the patchfile specifying any changes that should be applied |
| 170 |
# to the CachedSource before analyzing. |
| 171 |
PATCHFILE_NAME = "changes_for_analyzer.patch" |
| 172 |
|
| 173 |
# The list of checkers used during analyzes. |
| 174 |
# Currently, consists of all the non-experimental checkers, plus a few alpha |
| 175 |
# checkers we don't want to regress on. |
| 176 |
CHECKERS = ",".join([ |
| 177 |
"alpha.unix.SimpleStream", |
| 178 |
"alpha.security.taint", |
| 179 |
"cplusplus.NewDeleteLeaks", |
| 180 |
"core", |
| 181 |
"cplusplus", |
| 182 |
"deadcode", |
| 183 |
"security", |
| 184 |
"unix", |
| 185 |
"osx", |
| 186 |
"nullability" |
| 187 |
]) |
| 188 |
|
| 189 |
VERBOSE = 0 |
| 190 |
|
| 191 |
|
| 192 |
############################################################################### |
| 193 |
# Test harness logic. |
| 194 |
############################################################################### |
| 195 |
|
| 196 |
|
| 197 |
def run_cleanup_script(directory: str, build_log_file: IO): |
| 198 |
""" |
| 199 |
Run pre-processing script if any. |
| 200 |
""" |
| 201 |
cwd = os.path.join(directory, PATCHED_SOURCE_DIR_NAME) |
| 202 |
script_path = os.path.join(directory, CLEANUP_SCRIPT) |
| 203 |
|
| 204 |
utils.run_script(script_path, build_log_file, cwd, |
| 205 |
out=LOCAL.stdout, err=LOCAL.stderr, |
| 206 |
verbose=VERBOSE) |
| 207 |
|
| 208 |
|
| 209 |
class TestInfo(NamedTuple): |
| 210 |
""" |
| 211 |
Information about a project and settings for its analysis. |
| 212 |
""" |
| 213 |
project: ProjectInfo |
| 214 |
override_compiler: bool = False |
| 215 |
extra_analyzer_config: str = "" |
| 216 |
extra_checkers: str = "" |
| 217 |
is_reference_build: bool = False |
| 218 |
strictness: int = 0 |
| 219 |
|
| 220 |
|
| 221 |
# typing package doesn't have a separate type for Queue, but has a generic stub |
| 222 |
# We still want to have a type-safe checked project queue, for this reason, |
| 223 |
# we specify generic type for mypy. |
| 224 |
# |
| 225 |
# It is a common workaround for this situation: |
| 226 |
# https://mypy.readthedocs.io/en/stable/common_issues.html#using-classes-that-are-generic-in-stubs-but-not-at-runtime |
| 227 |
if TYPE_CHECKING: |
| 228 |
TestQueue = Queue[TestInfo] # this is only processed by mypy |
| 229 |
else: |
| 230 |
TestQueue = Queue # this will be executed at runtime |
| 231 |
|
| 232 |
|
| 233 |
class RegressionTester: |
| 234 |
""" |
| 235 |
A component aggregating all of the project testing. |
| 236 |
""" |
| 237 |
|
| 238 |
def __init__(self, jobs: int, projects: List[ProjectInfo], |
| 239 |
override_compiler: bool, extra_analyzer_config: str, |
| 240 |
extra_checkers: str, |
| 241 |
regenerate: bool, strictness: bool): |
| 242 |
self.jobs = jobs |
| 243 |
self.projects = projects |
| 244 |
self.override_compiler = override_compiler |
| 245 |
self.extra_analyzer_config = extra_analyzer_config |
| 246 |
self.extra_checkers = extra_checkers |
| 247 |
self.regenerate = regenerate |
| 248 |
self.strictness = strictness |
| 249 |
|
| 250 |
def test_all(self) -> bool: |
| 251 |
projects_to_test: List[TestInfo] = [] |
| 252 |
|
| 253 |
# Test the projects. |
| 254 |
for project in self.projects: |
| 255 |
projects_to_test.append( |
| 256 |
TestInfo(project, |
| 257 |
self.override_compiler, |
| 258 |
self.extra_analyzer_config, |
| 259 |
self.extra_checkers, |
| 260 |
self.regenerate, self.strictness)) |
| 261 |
if self.jobs <= 1: |
| 262 |
return self._single_threaded_test_all(projects_to_test) |
| 263 |
else: |
| 264 |
return self._multi_threaded_test_all(projects_to_test) |
| 265 |
|
| 266 |
def _single_threaded_test_all(self, |
| 267 |
projects_to_test: List[TestInfo]) -> bool: |
| 268 |
""" |
| 269 |
Run all projects. |
| 270 |
:return: whether tests have passed. |
| 271 |
""" |
| 272 |
success = True |
| 273 |
for project_info in projects_to_test: |
| 274 |
tester = ProjectTester(project_info) |
| 275 |
success &= tester.test() |
| 276 |
return success |
| 277 |
|
| 278 |
def _multi_threaded_test_all(self, |
| 279 |
projects_to_test: List[TestInfo]) -> bool: |
| 280 |
""" |
| 281 |
Run each project in a separate thread. |
| 282 |
|
| 283 |
This is OK despite GIL, as testing is blocked |
| 284 |
on launching external processes. |
| 285 |
|
| 286 |
:return: whether tests have passed. |
| 287 |
""" |
| 288 |
tasks_queue = TestQueue() |
| 289 |
|
| 290 |
for project_info in projects_to_test: |
| 291 |
tasks_queue.put(project_info) |
| 292 |
|
| 293 |
results_differ = threading.Event() |
| 294 |
failure_flag = threading.Event() |
| 295 |
|
| 296 |
for _ in range(self.jobs): |
| 297 |
T = TestProjectThread(tasks_queue, results_differ, failure_flag) |
| 298 |
T.start() |
| 299 |
|
| 300 |
# Required to handle Ctrl-C gracefully. |
| 301 |
while tasks_queue.unfinished_tasks: |
| 302 |
time.sleep(0.1) # Seconds. |
| 303 |
if failure_flag.is_set(): |
| 304 |
stderr("Test runner crashed\n") |
| 305 |
sys.exit(1) |
| 306 |
return not results_differ.is_set() |
| 307 |
|
| 308 |
|
| 309 |
class ProjectTester: |
| 310 |
""" |
| 311 |
A component aggregating testing for one project. |
| 312 |
""" |
| 313 |
|
| 314 |
def __init__(self, test_info: TestInfo, silent: bool = False): |
| 315 |
self.project = test_info.project |
| 316 |
self.override_compiler = test_info.override_compiler |
| 317 |
self.extra_analyzer_config = test_info.extra_analyzer_config |
| 318 |
self.extra_checkers = test_info.extra_checkers |
| 319 |
self.is_reference_build = test_info.is_reference_build |
| 320 |
self.strictness = test_info.strictness |
| 321 |
self.silent = silent |
| 322 |
|
| 323 |
def test(self) -> bool: |
| 324 |
""" |
| 325 |
Test a given project. |
| 326 |
:return tests_passed: Whether tests have passed according |
| 327 |
to the :param strictness: criteria. |
| 328 |
""" |
| 329 |
if not self.project.enabled: |
| 330 |
self.out( |
| 331 |
f" \n\n--- Skipping disabled project {self.project.name}\n") |
| 332 |
return True |
| 333 |
|
| 334 |
self.out(f" \n\n--- Building project {self.project.name}\n") |
| 335 |
|
| 336 |
start_time = time.time() |
| 337 |
|
| 338 |
project_dir = self.get_project_dir() |
| 339 |
self.vout(f" Build directory: {project_dir}.\n") |
| 340 |
|
| 341 |
# Set the build results directory. |
| 342 |
output_dir = self.get_output_dir() |
| 343 |
|
| 344 |
self.build(project_dir, output_dir) |
| 345 |
check_build(output_dir) |
| 346 |
|
| 347 |
if self.is_reference_build: |
| 348 |
cleanup_reference_results(output_dir) |
| 349 |
passed = True |
| 350 |
else: |
| 351 |
passed = run_cmp_results(project_dir, self.strictness) |
| 352 |
|
| 353 |
self.out(f"Completed tests for project {self.project.name} " |
| 354 |
f"(time: {time.time() - start_time:.2f}).\n") |
| 355 |
|
| 356 |
return passed |
| 357 |
|
| 358 |
def get_project_dir(self) -> str: |
| 359 |
return os.path.join(os.path.abspath(os.curdir), self.project.name) |
| 360 |
|
| 361 |
def get_output_dir(self) -> str: |
| 362 |
if self.is_reference_build: |
| 363 |
dirname = REF_PREFIX + OUTPUT_DIR_NAME |
| 364 |
else: |
| 365 |
dirname = OUTPUT_DIR_NAME |
| 366 |
|
| 367 |
return os.path.join(self.get_project_dir(), dirname) |
| 368 |
|
| 369 |
def build(self, directory: str, output_dir: str) -> Tuple[float, int]: |
| 370 |
build_log_path = get_build_log_path(output_dir) |
| 371 |
|
| 372 |
self.out(f"Log file: {build_log_path}\n") |
| 373 |
self.out(f"Output directory: {output_dir}\n") |
| 374 |
|
| 375 |
remove_log_file(output_dir) |
| 376 |
|
| 377 |
# Clean up scan build results. |
| 378 |
if os.path.exists(output_dir): |
| 379 |
self.vout(f" Removing old results: {output_dir}\n") |
| 380 |
|
| 381 |
shutil.rmtree(output_dir) |
| 382 |
|
| 383 |
assert(not os.path.exists(output_dir)) |
| 384 |
os.makedirs(os.path.join(output_dir, LOG_DIR_NAME)) |
| 385 |
|
| 386 |
# Build and analyze the project. |
| 387 |
with open(build_log_path, "w+") as build_log_file: |
| 388 |
if self.project.mode == 1: |
| 389 |
self._download_and_patch(directory, build_log_file) |
| 390 |
run_cleanup_script(directory, build_log_file) |
| 391 |
build_time, memory = self.scan_build(directory, output_dir, |
| 392 |
build_log_file) |
| 393 |
else: |
| 394 |
build_time, memory = self.analyze_preprocessed(directory, |
| 395 |
output_dir) |
| 396 |
|
| 397 |
if self.is_reference_build: |
| 398 |
run_cleanup_script(directory, build_log_file) |
| 399 |
normalize_reference_results(directory, output_dir, |
| 400 |
self.project.mode) |
| 401 |
|
| 402 |
self.out(f"Build complete (time: {utils.time_to_str(build_time)}, " |
| 403 |
f"peak memory: {utils.memory_to_str(memory)}). " |
| 404 |
f"See the log for more details: {build_log_path}\n") |
| 405 |
|
| 406 |
return build_time, memory |
| 407 |
|
| 408 |
def scan_build(self, directory: str, output_dir: str, |
| 409 |
build_log_file: IO) -> Tuple[float, int]: |
| 410 |
""" |
| 411 |
Build the project with scan-build by reading in the commands and |
| 412 |
prefixing them with the scan-build options. |
| 413 |
""" |
| 414 |
build_script_path = os.path.join(directory, BUILD_SCRIPT) |
| 415 |
if not os.path.exists(build_script_path): |
| 416 |
stderr(f"Error: build script is not defined: " |
| 417 |
f"{build_script_path}\n") |
| 418 |
sys.exit(1) |
| 419 |
|
| 420 |
all_checkers = CHECKERS |
| 421 |
if 'SA_ADDITIONAL_CHECKERS' in os.environ: |
| 422 |
all_checkers = (all_checkers + ',' + |
| 423 |
os.environ['SA_ADDITIONAL_CHECKERS']) |
| 424 |
if self.extra_checkers != "": |
| 425 |
all_checkers += "," + self.extra_checkers |
| 426 |
|
| 427 |
# Run scan-build from within the patched source directory. |
| 428 |
cwd = os.path.join(directory, PATCHED_SOURCE_DIR_NAME) |
| 429 |
|
| 430 |
options = f"--use-analyzer '{CLANG}' " |
| 431 |
options += f"-plist-html -o '{output_dir}' " |
| 432 |
options += f"-enable-checker {all_checkers} " |
| 433 |
options += "--keep-empty " |
| 434 |
options += f"-analyzer-config '{self.generate_config()}' " |
| 435 |
|
| 436 |
if self.override_compiler: |
| 437 |
options += "--override-compiler " |
| 438 |
|
| 439 |
extra_env: Dict[str, str] = {} |
| 440 |
|
| 441 |
execution_time = 0.0 |
| 442 |
peak_memory = 0 |
| 443 |
|
| 444 |
try: |
| 445 |
command_file = open(build_script_path, "r") |
| 446 |
command_prefix = "scan-build " + options + " " |
| 447 |
|
| 448 |
for command in command_file: |
| 449 |
command = command.strip() |
| 450 |
|
| 451 |
if len(command) == 0: |
| 452 |
continue |
| 453 |
|
| 454 |
# Custom analyzer invocation specified by project. |
| 455 |
# Communicate required information using environment variables |
| 456 |
# instead. |
| 457 |
if command == NO_PREFIX_CMD: |
| 458 |
command_prefix = "" |
| 459 |
extra_env['OUTPUT'] = output_dir |
| 460 |
extra_env['CC'] = CLANG |
| 461 |
extra_env['ANALYZER_CONFIG'] = self.generate_config() |
| 462 |
continue |
| 463 |
|
| 464 |
if command.startswith("#"): |
| 465 |
continue |
| 466 |
|
| 467 |
# If using 'make', auto imply a -jX argument |
| 468 |
# to speed up analysis. xcodebuild will |
| 469 |
# automatically use the maximum number of cores. |
| 470 |
if (command.startswith("make ") or command == "make") and \ |
| 471 |
"-j" not in command: |
| 472 |
command += f" -j{MAX_JOBS}" |
| 473 |
|
| 474 |
command_to_run = command_prefix + command |
| 475 |
|
| 476 |
self.vout(f" Executing: {command_to_run}\n") |
| 477 |
|
| 478 |
time, mem = utils.check_and_measure_call( |
| 479 |
command_to_run, cwd=cwd, |
| 480 |
stderr=build_log_file, |
| 481 |
stdout=build_log_file, |
| 482 |
env=dict(os.environ, **extra_env), |
| 483 |
shell=True) |
| 484 |
|
| 485 |
execution_time += time |
| 486 |
peak_memory = max(peak_memory, mem) |
| 487 |
|
| 488 |
except CalledProcessError: |
| 489 |
stderr("Error: scan-build failed. Its output was: \n") |
| 490 |
build_log_file.seek(0) |
| 491 |
shutil.copyfileobj(build_log_file, LOCAL.stderr) |
| 492 |
sys.exit(1) |
| 493 |
|
| 494 |
return execution_time, peak_memory |
| 495 |
|
| 496 |
def analyze_preprocessed(self, directory: str, |
| 497 |
output_dir: str) -> Tuple[float, int]: |
| 498 |
""" |
| 499 |
Run analysis on a set of preprocessed files. |
| 500 |
""" |
| 501 |
if os.path.exists(os.path.join(directory, BUILD_SCRIPT)): |
| 502 |
stderr(f"Error: The preprocessed files project " |
| 503 |
f"should not contain {BUILD_SCRIPT}\n") |
| 504 |
raise Exception() |
| 505 |
|
| 506 |
prefix = CLANG + " --analyze " |
| 507 |
|
| 508 |
prefix += "--analyzer-output plist " |
| 509 |
prefix += " -Xclang -analyzer-checker=" + CHECKERS |
| 510 |
prefix += " -fcxx-exceptions -fblocks " |
| 511 |
prefix += " -Xclang -analyzer-config " |
| 512 |
prefix += f"-Xclang {self.generate_config()} " |
| 513 |
|
| 514 |
if self.project.mode == 2: |
| 515 |
prefix += "-std=c++11 " |
| 516 |
|
| 517 |
plist_path = os.path.join(directory, output_dir, "date") |
| 518 |
fail_path = os.path.join(plist_path, "failures") |
| 519 |
os.makedirs(fail_path) |
| 520 |
|
| 521 |
execution_time = 0.0 |
| 522 |
peak_memory = 0 |
| 523 |
|
| 524 |
for full_file_name in glob.glob(directory + "/*"): |
| 525 |
file_name = os.path.basename(full_file_name) |
| 526 |
failed = False |
| 527 |
|
| 528 |
# Only run the analyzes on supported files. |
| 529 |
if utils.has_no_extension(file_name): |
| 530 |
continue |
| 531 |
if not utils.is_valid_single_input_file(file_name): |
| 532 |
stderr(f"Error: Invalid single input file {full_file_name}.\n") |
| 533 |
raise Exception() |
| 534 |
|
| 535 |
# Build and call the analyzer command. |
| 536 |
plist_basename = os.path.join(plist_path, file_name) |
| 537 |
output_option = f"-o '{plist_basename}.plist' " |
| 538 |
command = f"{prefix}{output_option}'{file_name}'" |
| 539 |
|
| 540 |
log_path = os.path.join(fail_path, file_name + ".stderr.txt") |
| 541 |
with open(log_path, "w+") as log_file: |
| 542 |
try: |
| 543 |
self.vout(f" Executing: {command}\n") |
| 544 |
|
| 545 |
time, mem = utils.check_and_measure_call( |
| 546 |
command, cwd=directory, stderr=log_file, |
| 547 |
stdout=log_file, shell=True) |
| 548 |
|
| 549 |
execution_time += time |
| 550 |
peak_memory = max(peak_memory, mem) |
| 551 |
|
| 552 |
except CalledProcessError as e: |
| 553 |
stderr(f"Error: Analyzes of {full_file_name} failed. " |
| 554 |
f"See {log_file.name} for details. " |
| 555 |
f"Error code {e.returncode}.\n") |
| 556 |
failed = True |
| 557 |
|
| 558 |
# If command did not fail, erase the log file. |
| 559 |
if not failed: |
| 560 |
os.remove(log_file.name) |
| 561 |
|
| 562 |
return execution_time, peak_memory |
| 563 |
|
| 564 |
def generate_config(self) -> str: |
| 565 |
out = "serialize-stats=true,stable-report-filename=true" |
| 566 |
|
| 567 |
if self.extra_analyzer_config: |
| 568 |
out += "," + self.extra_analyzer_config |
| 569 |
|
| 570 |
return out |
| 571 |
|
| 572 |
def _download_and_patch(self, directory: str, build_log_file: IO): |
| 573 |
""" |
| 574 |
Download the project and apply the local patchfile if it exists. |
| 575 |
""" |
| 576 |
cached_source = os.path.join(directory, CACHED_SOURCE_DIR_NAME) |
| 577 |
|
| 578 |
# If the we don't already have the cached source, run the project's |
| 579 |
# download script to download it. |
| 580 |
if not os.path.exists(cached_source): |
| 581 |
self._download(directory, build_log_file) |
| 582 |
if not os.path.exists(cached_source): |
| 583 |
stderr(f"Error: '{cached_source}' not found after download.\n") |
| 584 |
exit(1) |
| 585 |
|
| 586 |
patched_source = os.path.join(directory, PATCHED_SOURCE_DIR_NAME) |
| 587 |
|
| 588 |
# Remove potentially stale patched source. |
| 589 |
if os.path.exists(patched_source): |
| 590 |
shutil.rmtree(patched_source) |
| 591 |
|
| 592 |
# Copy the cached source and apply any patches to the copy. |
| 593 |
shutil.copytree(cached_source, patched_source, symlinks=True) |
| 594 |
self._apply_patch(directory, build_log_file) |
| 595 |
|
| 596 |
def _download(self, directory: str, build_log_file: IO): |
| 597 |
""" |
| 598 |
Run the script to download the project, if it exists. |
| 599 |
""" |
| 600 |
if self.project.source == DownloadType.GIT: |
| 601 |
self._download_from_git(directory, build_log_file) |
| 602 |
elif self.project.source == DownloadType.ZIP: |
| 603 |
self._unpack_zip(directory, build_log_file) |
| 604 |
elif self.project.source == DownloadType.SCRIPT: |
| 605 |
self._run_download_script(directory, build_log_file) |
| 606 |
else: |
| 607 |
raise ValueError( |
| 608 |
f"Unknown source type '{self.project.source}' is found " |
| 609 |
f"for the '{self.project.name}' project") |
| 610 |
|
| 611 |
def _download_from_git(self, directory: str, build_log_file: IO): |
| 612 |
repo = self.project.origin |
| 613 |
cached_source = os.path.join(directory, CACHED_SOURCE_DIR_NAME) |
| 614 |
|
| 615 |
check_call(f"git clone --recursive {repo} {cached_source}", |
| 616 |
cwd=directory, stderr=build_log_file, |
| 617 |
stdout=build_log_file, shell=True) |
| 618 |
check_call(f"git checkout --quiet {self.project.commit}", |
| 619 |
cwd=cached_source, stderr=build_log_file, |
| 620 |
stdout=build_log_file, shell=True) |
| 621 |
|
| 622 |
def _unpack_zip(self, directory: str, build_log_file: IO): |
| 623 |
zip_files = list(glob.glob(directory + "/*.zip")) |
| 624 |
|
| 625 |
if len(zip_files) == 0: |
| 626 |
raise ValueError( |
| 627 |
f"Couldn't find any zip files to unpack for the " |
| 628 |
f"'{self.project.name}' project") |
| 629 |
|
| 630 |
if len(zip_files) > 1: |
| 631 |
raise ValueError( |
| 632 |
f"Couldn't decide which of the zip files ({zip_files}) " |
| 633 |
f"for the '{self.project.name}' project to unpack") |
| 634 |
|
| 635 |
with zipfile.ZipFile(zip_files[0], "r") as zip_file: |
| 636 |
zip_file.extractall(os.path.join(directory, |
| 637 |
CACHED_SOURCE_DIR_NAME)) |
| 638 |
|
| 639 |
@staticmethod |
| 640 |
def _run_download_script(directory: str, build_log_file: IO): |
| 641 |
script_path = os.path.join(directory, DOWNLOAD_SCRIPT) |
| 642 |
utils.run_script(script_path, build_log_file, directory, |
| 643 |
out=LOCAL.stdout, err=LOCAL.stderr, |
| 644 |
verbose=VERBOSE) |
| 645 |
|
| 646 |
def _apply_patch(self, directory: str, build_log_file: IO): |
| 647 |
patchfile_path = os.path.join(directory, PATCHFILE_NAME) |
| 648 |
patched_source = os.path.join(directory, PATCHED_SOURCE_DIR_NAME) |
| 649 |
|
| 650 |
if not os.path.exists(patchfile_path): |
| 651 |
self.out(" No local patches.\n") |
| 652 |
return |
| 653 |
|
| 654 |
self.out(" Applying patch.\n") |
| 655 |
try: |
| 656 |
check_call(f"patch -p1 < '{patchfile_path}'", |
| 657 |
cwd=patched_source, |
| 658 |
stderr=build_log_file, |
| 659 |
stdout=build_log_file, |
| 660 |
shell=True) |
| 661 |
|
| 662 |
except CalledProcessError: |
| 663 |
stderr(f"Error: Patch failed. " |
| 664 |
f"See {build_log_file.name} for details.\n") |
| 665 |
sys.exit(1) |
| 666 |
|
| 667 |
def out(self, what: str): |
| 668 |
if not self.silent: |
| 669 |
stdout(what) |
| 670 |
|
| 671 |
def vout(self, what: str): |
| 672 |
if VERBOSE >= 1: |
| 673 |
self.out(what) |
| 674 |
|
| 675 |
|
| 676 |
class TestProjectThread(threading.Thread): |
| 677 |
def __init__(self, tasks_queue: TestQueue, |
| 678 |
results_differ: threading.Event, |
| 679 |
failure_flag: threading.Event): |
| 680 |
""" |
| 681 |
:param results_differ: Used to signify that results differ from |
| 682 |
the canonical ones. |
| 683 |
:param failure_flag: Used to signify a failure during the run. |
| 684 |
""" |
| 685 |
self.tasks_queue = tasks_queue |
| 686 |
self.results_differ = results_differ |
| 687 |
self.failure_flag = failure_flag |
| 688 |
super().__init__() |
| 689 |
|
| 690 |
# Needed to gracefully handle interrupts with Ctrl-C |
| 691 |
self.daemon = True |
| 692 |
|
| 693 |
def run(self): |
| 694 |
while not self.tasks_queue.empty(): |
| 695 |
try: |
| 696 |
test_info = self.tasks_queue.get() |
| 697 |
init_logger(test_info.project.name) |
| 698 |
|
| 699 |
tester = ProjectTester(test_info) |
| 700 |
if not tester.test(): |
| 701 |
self.results_differ.set() |
| 702 |
|
| 703 |
self.tasks_queue.task_done() |
| 704 |
|
| 705 |
except BaseException: |
| 706 |
self.failure_flag.set() |
| 707 |
raise |
| 708 |
|
| 709 |
|
| 710 |
############################################################################### |
| 711 |
# Utility functions. |
| 712 |
############################################################################### |
| 713 |
|
| 714 |
|
| 715 |
def check_build(output_dir: str): |
| 716 |
""" |
| 717 |
Given the scan-build output directory, checks if the build failed |
| 718 |
(by searching for the failures directories). If there are failures, it |
| 719 |
creates a summary file in the output directory. |
| 720 |
|
| 721 |
""" |
| 722 |
# Check if there are failures. |
| 723 |
failures = glob.glob(output_dir + "/*/failures/*.stderr.txt") |
| 724 |
total_failed = len(failures) |
| 725 |
|
| 726 |
if total_failed == 0: |
| 727 |
clean_up_empty_plists(output_dir) |
| 728 |
clean_up_empty_folders(output_dir) |
| 729 |
|
| 730 |
plists = glob.glob(output_dir + "/*/*.plist") |
| 731 |
stdout(f"Number of bug reports " |
| 732 |
f"(non-empty plist files) produced: {len(plists)}\n") |
| 733 |
return |
| 734 |
|
| 735 |
stderr("Error: analysis failed.\n") |
| 736 |
stderr(f"Total of {total_failed} failures discovered.\n") |
| 737 |
|
| 738 |
if total_failed > NUM_OF_FAILURES_IN_SUMMARY: |
| 739 |
stderr(f"See the first {NUM_OF_FAILURES_IN_SUMMARY} below.\n") |
| 740 |
|
| 741 |
for index, failed_log_path in enumerate(failures, start=1): |
| 742 |
if index >= NUM_OF_FAILURES_IN_SUMMARY: |
| 743 |
break |
| 744 |
|
| 745 |
stderr(f"\n-- Error #{index} -----------\n") |
| 746 |
|
| 747 |
with open(failed_log_path, "r") as failed_log: |
| 748 |
shutil.copyfileobj(failed_log, LOCAL.stdout) |
| 749 |
|
| 750 |
if total_failed > NUM_OF_FAILURES_IN_SUMMARY: |
| 751 |
stderr("See the results folder for more.") |
| 752 |
|
| 753 |
sys.exit(1) |
| 754 |
|
| 755 |
|
| 756 |
def cleanup_reference_results(output_dir: str): |
| 757 |
""" |
| 758 |
Delete html, css, and js files from reference results. These can |
| 759 |
include multiple copies of the benchmark source and so get very large. |
| 760 |
""" |
| 761 |
extensions = ["html", "css", "js"] |
| 762 |
|
| 763 |
for extension in extensions: |
| 764 |
for file_to_rm in glob.glob(f"{output_dir}/*/*.{extension}"): |
| 765 |
file_to_rm = os.path.join(output_dir, file_to_rm) |
| 766 |
os.remove(file_to_rm) |
| 767 |
|
| 768 |
# Remove the log file. It leaks absolute path names. |
| 769 |
remove_log_file(output_dir) |
| 770 |
|
| 771 |
|
| 772 |
def run_cmp_results(directory: str, strictness: int = 0) -> bool: |
| 773 |
""" |
| 774 |
Compare the warnings produced by scan-build. |
| 775 |
strictness defines the success criteria for the test: |
| 776 |
0 - success if there are no crashes or analyzer failure. |
| 777 |
1 - success if there are no difference in the number of reported bugs. |
| 778 |
2 - success if all the bug reports are identical. |
| 779 |
|
| 780 |
:return success: Whether tests pass according to the strictness |
| 781 |
criteria. |
| 782 |
""" |
| 783 |
tests_passed = True |
| 784 |
start_time = time.time() |
| 785 |
|
| 786 |
ref_dir = os.path.join(directory, REF_PREFIX + OUTPUT_DIR_NAME) |
| 787 |
new_dir = os.path.join(directory, OUTPUT_DIR_NAME) |
| 788 |
|
| 789 |
# We have to go one level down the directory tree. |
| 790 |
ref_list = glob.glob(ref_dir + "/*") |
| 791 |
new_list = glob.glob(new_dir + "/*") |
| 792 |
|
| 793 |
# Log folders are also located in the results dir, so ignore them. |
| 794 |
ref_log_dir = os.path.join(ref_dir, LOG_DIR_NAME) |
| 795 |
if ref_log_dir in ref_list: |
| 796 |
ref_list.remove(ref_log_dir) |
| 797 |
new_list.remove(os.path.join(new_dir, LOG_DIR_NAME)) |
| 798 |
|
| 799 |
if len(ref_list) != len(new_list): |
| 800 |
stderr(f"Mismatch in number of results folders: " |
| 801 |
f"{ref_list} vs {new_list}") |
| 802 |
sys.exit(1) |
| 803 |
|
| 804 |
# There might be more then one folder underneath - one per each scan-build |
| 805 |
# command (Ex: one for configure and one for make). |
| 806 |
if len(ref_list) > 1: |
| 807 |
# Assume that the corresponding folders have the same names. |
| 808 |
ref_list.sort() |
| 809 |
new_list.sort() |
| 810 |
|
| 811 |
# Iterate and find the differences. |
| 812 |
num_diffs = 0 |
| 813 |
for ref_dir, new_dir in zip(ref_list, new_list): |
| 814 |
assert(ref_dir != new_dir) |
| 815 |
|
| 816 |
if VERBOSE >= 1: |
| 817 |
stdout(f" Comparing Results: {ref_dir} {new_dir}\n") |
| 818 |
|
| 819 |
patched_source = os.path.join(directory, PATCHED_SOURCE_DIR_NAME) |
| 820 |
|
| 821 |
ref_results = CmpRuns.ResultsDirectory(ref_dir) |
| 822 |
new_results = CmpRuns.ResultsDirectory(new_dir, patched_source) |
| 823 |
|
| 824 |
# Scan the results, delete empty plist files. |
| 825 |
num_diffs, reports_in_ref, reports_in_new = \ |
| 826 |
CmpRuns.dump_scan_build_results_diff(ref_results, new_results, |
| 827 |
delete_empty=False, |
| 828 |
out=LOCAL.stdout) |
| 829 |
|
| 830 |
if num_diffs > 0: |
| 831 |
stdout(f"Warning: {num_diffs} differences in diagnostics.\n") |
| 832 |
|
| 833 |
if strictness >= 2 and num_diffs > 0: |
| 834 |
stdout("Error: Diffs found in strict mode (2).\n") |
| 835 |
tests_passed = False |
| 836 |
|
| 837 |
elif strictness >= 1 and reports_in_ref != reports_in_new: |
| 838 |
stdout("Error: The number of results are different " |
| 839 |
" strict mode (1).\n") |
| 840 |
tests_passed = False |
| 841 |
|
| 842 |
stdout(f"Diagnostic comparison complete " |
| 843 |
f"(time: {time.time() - start_time:.2f}).\n") |
| 844 |
|
| 845 |
return tests_passed |
| 846 |
|
| 847 |
|
| 848 |
def normalize_reference_results(directory: str, output_dir: str, |
| 849 |
build_mode: int): |
| 850 |
""" |
| 851 |
Make the absolute paths relative in the reference results. |
| 852 |
""" |
| 853 |
for dir_path, _, filenames in os.walk(output_dir): |
| 854 |
for filename in filenames: |
| 855 |
if not filename.endswith('plist'): |
| 856 |
continue |
| 857 |
|
| 858 |
plist = os.path.join(dir_path, filename) |
| 859 |
with open(plist, "rb") as plist_file: |
| 860 |
data = plistlib.load(plist_file) |
| 861 |
path_prefix = directory |
| 862 |
|
| 863 |
if build_mode == 1: |
| 864 |
path_prefix = os.path.join(directory, PATCHED_SOURCE_DIR_NAME) |
| 865 |
|
| 866 |
paths = [source[len(path_prefix) + 1:] |
| 867 |
if source.startswith(path_prefix) else source |
| 868 |
for source in data['files']] |
| 869 |
data['files'] = paths |
| 870 |
|
| 871 |
# Remove transient fields which change from run to run. |
| 872 |
for diagnostic in data['diagnostics']: |
| 873 |
if 'HTMLDiagnostics_files' in diagnostic: |
| 874 |
diagnostic.pop('HTMLDiagnostics_files') |
| 875 |
|
| 876 |
if 'clang_version' in data: |
| 877 |
data.pop('clang_version') |
| 878 |
|
| 879 |
with open(plist, "wb") as plist_file: |
| 880 |
plistlib.dump(data, plist_file) |
| 881 |
|
| 882 |
|
| 883 |
def get_build_log_path(output_dir: str) -> str: |
| 884 |
return os.path.join(output_dir, LOG_DIR_NAME, BUILD_LOG_NAME) |
| 885 |
|
| 886 |
|
| 887 |
def remove_log_file(output_dir: str): |
| 888 |
build_log_path = get_build_log_path(output_dir) |
| 889 |
|
| 890 |
# Clean up the log file. |
| 891 |
if os.path.exists(build_log_path): |
| 892 |
if VERBOSE >= 1: |
| 893 |
stdout(f" Removing log file: {build_log_path}\n") |
| 894 |
|
| 895 |
os.remove(build_log_path) |
| 896 |
|
| 897 |
|
| 898 |
def clean_up_empty_plists(output_dir: str): |
| 899 |
""" |
| 900 |
A plist file is created for each call to the analyzer(each source file). |
| 901 |
We are only interested on the once that have bug reports, |
| 902 |
so delete the rest. |
| 903 |
""" |
| 904 |
for plist in glob.glob(output_dir + "/*/*.plist"): |
| 905 |
plist = os.path.join(output_dir, plist) |
| 906 |
|
| 907 |
try: |
| 908 |
with open(plist, "rb") as plist_file: |
| 909 |
data = plistlib.load(plist_file) |
| 910 |
# Delete empty reports. |
| 911 |
if not data['files']: |
| 912 |
os.remove(plist) |
| 913 |
continue |
| 914 |
|
| 915 |
except InvalidFileException as e: |
| 916 |
stderr(f"Error parsing plist file {plist}: {str(e)}") |
| 917 |
continue |
| 918 |
|
| 919 |
|
| 920 |
def clean_up_empty_folders(output_dir: str): |
| 921 |
""" |
| 922 |
Remove empty folders from results, as git would not store them. |
| 923 |
""" |
| 924 |
subdirs = glob.glob(output_dir + "/*") |
| 925 |
for subdir in subdirs: |
| 926 |
if not os.listdir(subdir): |
| 927 |
os.removedirs(subdir) |
| 928 |
|
| 929 |
|
| 930 |
if __name__ == "__main__": |
| 931 |
print("SATestBuild.py should not be used on its own.") |
| 932 |
print("Please use 'SATest.py build' instead") |
| 933 |
sys.exit(1) |