| 1 |
#!/usr/bin/env python |
| 2 |
|
| 3 |
import argparse |
| 4 |
import sys |
| 5 |
import os |
| 6 |
|
| 7 |
from subprocess import call |
| 8 |
|
| 9 |
SCRIPTS_DIR = os.path.dirname(os.path.realpath(__file__)) |
| 10 |
PROJECTS_DIR = os.path.join(SCRIPTS_DIR, "projects") |
| 11 |
DEFAULT_LLVM_DIR = os.path.realpath(os.path.join(SCRIPTS_DIR, |
| 12 |
os.path.pardir, |
| 13 |
os.path.pardir, |
| 14 |
os.path.pardir)) |
| 15 |
|
| 16 |
|
| 17 |
def add(parser, args): |
| 18 |
import SATestAdd |
| 19 |
from ProjectMap import ProjectInfo |
| 20 |
|
| 21 |
if args.source == "git" and (args.origin == "" or args.commit == ""): |
| 22 |
parser.error( |
| 23 |
"Please provide both --origin and --commit if source is 'git'") |
| 24 |
|
| 25 |
if args.source != "git" and (args.origin != "" or args.commit != ""): |
| 26 |
parser.error("Options --origin and --commit don't make sense when " |
| 27 |
"source is not 'git'") |
| 28 |
|
| 29 |
project = ProjectInfo(args.name[0], args.mode, args.source, args.origin, |
| 30 |
args.commit) |
| 31 |
|
| 32 |
SATestAdd.add_new_project(project) |
| 33 |
|
| 34 |
|
| 35 |
def build(parser, args): |
| 36 |
import SATestBuild |
| 37 |
|
| 38 |
SATestBuild.VERBOSE = args.verbose |
| 39 |
|
| 40 |
projects = get_projects(parser, args) |
| 41 |
tester = SATestBuild.RegressionTester(args.jobs, |
| 42 |
projects, |
| 43 |
args.override_compiler, |
| 44 |
args.extra_analyzer_config, |
| 45 |
args.extra_checkers, |
| 46 |
args.regenerate, |
| 47 |
args.strictness) |
| 48 |
tests_passed = tester.test_all() |
| 49 |
|
| 50 |
if not tests_passed: |
| 51 |
sys.stderr.write("ERROR: Tests failed.\n") |
| 52 |
sys.exit(42) |
| 53 |
|
| 54 |
|
| 55 |
def compare(parser, args): |
| 56 |
import CmpRuns |
| 57 |
|
| 58 |
choices = [CmpRuns.HistogramType.RELATIVE.value, |
| 59 |
CmpRuns.HistogramType.LOG_RELATIVE.value, |
| 60 |
CmpRuns.HistogramType.ABSOLUTE.value] |
| 61 |
|
| 62 |
if args.histogram is not None and args.histogram not in choices: |
| 63 |
parser.error("Incorrect histogram type, available choices are {}" |
| 64 |
.format(choices)) |
| 65 |
|
| 66 |
dir_old = CmpRuns.ResultsDirectory(args.old[0], args.root_old) |
| 67 |
dir_new = CmpRuns.ResultsDirectory(args.new[0], args.root_new) |
| 68 |
|
| 69 |
CmpRuns.dump_scan_build_results_diff(dir_old, dir_new, |
| 70 |
show_stats=args.show_stats, |
| 71 |
stats_only=args.stats_only, |
| 72 |
histogram=args.histogram, |
| 73 |
verbose_log=args.verbose_log) |
| 74 |
|
| 75 |
|
| 76 |
def update(parser, args): |
| 77 |
import SATestUpdateDiffs |
| 78 |
from ProjectMap import ProjectMap |
| 79 |
|
| 80 |
project_map = ProjectMap() |
| 81 |
for project in project_map.projects: |
| 82 |
SATestUpdateDiffs.update_reference_results(project, args.git) |
| 83 |
|
| 84 |
|
| 85 |
def benchmark(parser, args): |
| 86 |
from SATestBenchmark import Benchmark |
| 87 |
|
| 88 |
projects = get_projects(parser, args) |
| 89 |
benchmark = Benchmark(projects, args.iterations, args.output) |
| 90 |
benchmark.run() |
| 91 |
|
| 92 |
|
| 93 |
def benchmark_compare(parser, args): |
| 94 |
import SATestBenchmark |
| 95 |
SATestBenchmark.compare(args.old, args.new, args.output) |
| 96 |
|
| 97 |
|
| 98 |
def get_projects(parser, args): |
| 99 |
from ProjectMap import ProjectMap, Size |
| 100 |
|
| 101 |
project_map = ProjectMap() |
| 102 |
projects = project_map.projects |
| 103 |
|
| 104 |
def filter_projects(projects, predicate, force=False): |
| 105 |
return [project.with_fields(enabled=(force or project.enabled) and |
| 106 |
predicate(project)) |
| 107 |
for project in projects] |
| 108 |
|
| 109 |
if args.projects: |
| 110 |
projects_arg = args.projects.split(",") |
| 111 |
available_projects = [project.name |
| 112 |
for project in projects] |
| 113 |
|
| 114 |
# validate that given projects are present in the project map file |
| 115 |
for manual_project in projects_arg: |
| 116 |
if manual_project not in available_projects: |
| 117 |
parser.error("Project '{project}' is not found in " |
| 118 |
"the project map file. Available projects are " |
| 119 |
"{all}.".format(project=manual_project, |
| 120 |
all=available_projects)) |
| 121 |
|
| 122 |
projects = filter_projects(projects, lambda project: |
| 123 |
project.name in projects_arg, |
| 124 |
force=True) |
| 125 |
|
| 126 |
try: |
| 127 |
max_size = Size.from_str(args.max_size) |
| 128 |
except ValueError as e: |
| 129 |
parser.error("{}".format(e)) |
| 130 |
|
| 131 |
projects = filter_projects(projects, lambda project: |
| 132 |
project.size <= max_size) |
| 133 |
|
| 134 |
return projects |
| 135 |
|
| 136 |
|
| 137 |
def docker(parser, args): |
| 138 |
if len(args.rest) > 0: |
| 139 |
if args.rest[0] != "--": |
| 140 |
parser.error("REST arguments should start with '--'") |
| 141 |
args.rest = args.rest[1:] |
| 142 |
|
| 143 |
if args.build_image: |
| 144 |
docker_build_image() |
| 145 |
elif args.shell: |
| 146 |
docker_shell(args) |
| 147 |
else: |
| 148 |
sys.exit(docker_run(args, ' '.join(args.rest))) |
| 149 |
|
| 150 |
|
| 151 |
def docker_build_image(): |
| 152 |
sys.exit(call("docker build --tag satest-image {}".format(SCRIPTS_DIR), |
| 153 |
shell=True)) |
| 154 |
|
| 155 |
|
| 156 |
def docker_shell(args): |
| 157 |
try: |
| 158 |
# First we need to start the docker container in a waiting mode, |
| 159 |
# so it doesn't do anything, but most importantly keeps working |
| 160 |
# while the shell session is in progress. |
| 161 |
docker_run(args, "--wait", "--detach") |
| 162 |
# Since the docker container is running, we can actually connect to it |
| 163 |
call("docker exec -it satest bash", shell=True) |
| 164 |
|
| 165 |
except KeyboardInterrupt: |
| 166 |
pass |
| 167 |
|
| 168 |
finally: |
| 169 |
docker_cleanup() |
| 170 |
|
| 171 |
|
| 172 |
def docker_run(args, command, docker_args=""): |
| 173 |
try: |
| 174 |
return call("docker run --rm --name satest " |
| 175 |
"-v {llvm}:/llvm-project " |
| 176 |
"-v {build}:/build " |
| 177 |
"-v {clang}:/analyzer " |
| 178 |
"-v {scripts}:/scripts " |
| 179 |
"-v {projects}:/projects " |
| 180 |
"{docker_args} " |
| 181 |
"satest-image:latest {command}" |
| 182 |
.format(llvm=args.llvm_project_dir, |
| 183 |
build=args.build_dir, |
| 184 |
clang=args.clang_dir, |
| 185 |
scripts=SCRIPTS_DIR, |
| 186 |
projects=PROJECTS_DIR, |
| 187 |
docker_args=docker_args, |
| 188 |
command=command), |
| 189 |
shell=True) |
| 190 |
|
| 191 |
except KeyboardInterrupt: |
| 192 |
docker_cleanup() |
| 193 |
|
| 194 |
|
| 195 |
def docker_cleanup(): |
| 196 |
print("Please wait for docker to clean up") |
| 197 |
call("docker stop satest", shell=True) |
| 198 |
|
| 199 |
|
| 200 |
def main(): |
| 201 |
parser = argparse.ArgumentParser() |
| 202 |
subparsers = parser.add_subparsers() |
| 203 |
|
| 204 |
# add subcommand |
| 205 |
add_parser = subparsers.add_parser( |
| 206 |
"add", |
| 207 |
help="Add a new project for the analyzer testing.") |
| 208 |
# TODO: Add an option not to build. |
| 209 |
# TODO: Set the path to the Repository directory. |
| 210 |
add_parser.add_argument("name", nargs=1, help="Name of the new project") |
| 211 |
add_parser.add_argument("--mode", action="store", default=1, type=int, |
| 212 |
choices=[0, 1, 2], |
| 213 |
help="Build mode: 0 for single file project, " |
| 214 |
"1 for scan_build, " |
| 215 |
"2 for single file c++11 project") |
| 216 |
add_parser.add_argument("--source", action="store", default="script", |
| 217 |
choices=["script", "git", "zip"], |
| 218 |
help="Source type of the new project: " |
| 219 |
"'git' for getting from git " |
| 220 |
"(please provide --origin and --commit), " |
| 221 |
"'zip' for unpacking source from a zip file, " |
| 222 |
"'script' for downloading source by running " |
| 223 |
"a custom script") |
| 224 |
add_parser.add_argument("--origin", action="store", default="", |
| 225 |
help="Origin link for a git repository") |
| 226 |
add_parser.add_argument("--commit", action="store", default="", |
| 227 |
help="Git hash for a commit to checkout") |
| 228 |
add_parser.set_defaults(func=add) |
| 229 |
|
| 230 |
# build subcommand |
| 231 |
build_parser = subparsers.add_parser( |
| 232 |
"build", |
| 233 |
help="Build projects from the project map and compare results with " |
| 234 |
"the reference.") |
| 235 |
build_parser.add_argument("--strictness", dest="strictness", |
| 236 |
type=int, default=0, |
| 237 |
help="0 to fail on runtime errors, 1 to fail " |
| 238 |
"when the number of found bugs are different " |
| 239 |
"from the reference, 2 to fail on any " |
| 240 |
"difference from the reference. Default is 0.") |
| 241 |
build_parser.add_argument("-r", dest="regenerate", action="store_true", |
| 242 |
default=False, |
| 243 |
help="Regenerate reference output.") |
| 244 |
build_parser.add_argument("--override-compiler", action="store_true", |
| 245 |
default=False, help="Call scan-build with " |
| 246 |
"--override-compiler option.") |
| 247 |
build_parser.add_argument("-j", "--jobs", dest="jobs", |
| 248 |
type=int, default=0, |
| 249 |
help="Number of projects to test concurrently") |
| 250 |
build_parser.add_argument("--extra-analyzer-config", |
| 251 |
dest="extra_analyzer_config", type=str, |
| 252 |
default="", |
| 253 |
help="Arguments passed to to -analyzer-config") |
| 254 |
build_parser.add_argument("--extra-checkers", |
| 255 |
dest="extra_checkers", type=str, |
| 256 |
default="", |
| 257 |
help="Extra checkers to enable") |
| 258 |
build_parser.add_argument("--projects", action="store", default="", |
| 259 |
help="Comma-separated list of projects to test") |
| 260 |
build_parser.add_argument("--max-size", action="store", default=None, |
| 261 |
help="Maximum size for the projects to test") |
| 262 |
build_parser.add_argument("-v", "--verbose", action="count", default=0) |
| 263 |
build_parser.set_defaults(func=build) |
| 264 |
|
| 265 |
# compare subcommand |
| 266 |
cmp_parser = subparsers.add_parser( |
| 267 |
"compare", |
| 268 |
help="Comparing two static analyzer runs in terms of " |
| 269 |
"reported warnings and execution time statistics.") |
| 270 |
cmp_parser.add_argument("--root-old", dest="root_old", |
| 271 |
help="Prefix to ignore on source files for " |
| 272 |
"OLD directory", |
| 273 |
action="store", type=str, default="") |
| 274 |
cmp_parser.add_argument("--root-new", dest="root_new", |
| 275 |
help="Prefix to ignore on source files for " |
| 276 |
"NEW directory", |
| 277 |
action="store", type=str, default="") |
| 278 |
cmp_parser.add_argument("--verbose-log", dest="verbose_log", |
| 279 |
help="Write additional information to LOG " |
| 280 |
"[default=None]", |
| 281 |
action="store", type=str, default=None, |
| 282 |
metavar="LOG") |
| 283 |
cmp_parser.add_argument("--stats-only", action="store_true", |
| 284 |
dest="stats_only", default=False, |
| 285 |
help="Only show statistics on reports") |
| 286 |
cmp_parser.add_argument("--show-stats", action="store_true", |
| 287 |
dest="show_stats", default=False, |
| 288 |
help="Show change in statistics") |
| 289 |
cmp_parser.add_argument("--histogram", action="store", default=None, |
| 290 |
help="Show histogram of paths differences. " |
| 291 |
"Requires matplotlib") |
| 292 |
cmp_parser.add_argument("old", nargs=1, help="Directory with old results") |
| 293 |
cmp_parser.add_argument("new", nargs=1, help="Directory with new results") |
| 294 |
cmp_parser.set_defaults(func=compare) |
| 295 |
|
| 296 |
# update subcommand |
| 297 |
upd_parser = subparsers.add_parser( |
| 298 |
"update", |
| 299 |
help="Update static analyzer reference results based on the previous " |
| 300 |
"run of SATest build. Assumes that SATest build was just run.") |
| 301 |
upd_parser.add_argument("--git", action="store_true", |
| 302 |
help="Stage updated results using git.") |
| 303 |
upd_parser.set_defaults(func=update) |
| 304 |
|
| 305 |
# docker subcommand |
| 306 |
dock_parser = subparsers.add_parser( |
| 307 |
"docker", |
| 308 |
help="Run regression system in the docker.") |
| 309 |
|
| 310 |
dock_parser.add_argument("--build-image", action="store_true", |
| 311 |
help="Build docker image for running tests.") |
| 312 |
dock_parser.add_argument("--shell", action="store_true", |
| 313 |
help="Start a shell on docker.") |
| 314 |
dock_parser.add_argument("--llvm-project-dir", action="store", |
| 315 |
default=DEFAULT_LLVM_DIR, |
| 316 |
help="Path to LLVM source code. Defaults " |
| 317 |
"to the repo where this script is located. ") |
| 318 |
dock_parser.add_argument("--build-dir", action="store", default="", |
| 319 |
help="Path to a directory where docker should " |
| 320 |
"build LLVM code.") |
| 321 |
dock_parser.add_argument("--clang-dir", action="store", default="", |
| 322 |
help="Path to find/install LLVM installation.") |
| 323 |
dock_parser.add_argument("rest", nargs=argparse.REMAINDER, default=[], |
| 324 |
help="Additional args that will be forwarded " |
| 325 |
"to the docker's entrypoint.") |
| 326 |
dock_parser.set_defaults(func=docker) |
| 327 |
|
| 328 |
# benchmark subcommand |
| 329 |
bench_parser = subparsers.add_parser( |
| 330 |
"benchmark", |
| 331 |
help="Run benchmarks by building a set of projects multiple times.") |
| 332 |
|
| 333 |
bench_parser.add_argument("-i", "--iterations", action="store", |
| 334 |
type=int, default=20, |
| 335 |
help="Number of iterations for building each " |
| 336 |
"project.") |
| 337 |
bench_parser.add_argument("-o", "--output", action="store", |
| 338 |
default="benchmark.csv", |
| 339 |
help="Output csv file for the benchmark results") |
| 340 |
bench_parser.add_argument("--projects", action="store", default="", |
| 341 |
help="Comma-separated list of projects to test") |
| 342 |
bench_parser.add_argument("--max-size", action="store", default=None, |
| 343 |
help="Maximum size for the projects to test") |
| 344 |
bench_parser.set_defaults(func=benchmark) |
| 345 |
|
| 346 |
bench_subparsers = bench_parser.add_subparsers() |
| 347 |
bench_compare_parser = bench_subparsers.add_parser( |
| 348 |
"compare", |
| 349 |
help="Compare benchmark runs.") |
| 350 |
bench_compare_parser.add_argument("--old", action="store", required=True, |
| 351 |
help="Benchmark reference results to " |
| 352 |
"compare agains.") |
| 353 |
bench_compare_parser.add_argument("--new", action="store", required=True, |
| 354 |
help="New benchmark results to check.") |
| 355 |
bench_compare_parser.add_argument("-o", "--output", |
| 356 |
action="store", required=True, |
| 357 |
help="Output file for plots.") |
| 358 |
bench_compare_parser.set_defaults(func=benchmark_compare) |
| 359 |
|
| 360 |
args = parser.parse_args() |
| 361 |
args.func(parser, args) |
| 362 |
|
| 363 |
|
| 364 |
if __name__ == "__main__": |
| 365 |
main() |