| 1 |
#!/usr/bin/env python3 |
| 2 |
# |
| 3 |
# ===- clang-tidy-diff.py - ClangTidy Diff Checker -----------*- python -*--===# |
| 4 |
# |
| 5 |
# Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. |
| 6 |
# See https://llvm.org/LICENSE.txt for license information. |
| 7 |
# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception |
| 8 |
# |
| 9 |
# ===-----------------------------------------------------------------------===# |
| 10 |
|
| 11 |
r""" |
| 12 |
ClangTidy Diff Checker |
| 13 |
====================== |
| 14 |
|
| 15 |
This script reads input from a unified diff, runs clang-tidy on all changed |
| 16 |
files and outputs clang-tidy warnings in changed lines only. This is useful to |
| 17 |
detect clang-tidy regressions in the lines touched by a specific patch. |
| 18 |
Example usage for git/svn users: |
| 19 |
|
| 20 |
git diff -U0 HEAD^ | clang-tidy-diff.py -p1 |
| 21 |
svn diff --diff-cmd=diff -x-U0 | \ |
| 22 |
clang-tidy-diff.py -fix -checks=-*,modernize-use-override |
| 23 |
|
| 24 |
""" |
| 25 |
|
| 26 |
import argparse |
| 27 |
import glob |
| 28 |
import json |
| 29 |
import multiprocessing |
| 30 |
import os |
| 31 |
import re |
| 32 |
import shutil |
| 33 |
import subprocess |
| 34 |
import sys |
| 35 |
import tempfile |
| 36 |
import threading |
| 37 |
import traceback |
| 38 |
|
| 39 |
try: |
| 40 |
import yaml |
| 41 |
except ImportError: |
| 42 |
yaml = None |
| 43 |
|
| 44 |
is_py2 = sys.version[0] == "2" |
| 45 |
|
| 46 |
if is_py2: |
| 47 |
import Queue as queue |
| 48 |
else: |
| 49 |
import queue as queue |
| 50 |
|
| 51 |
|
| 52 |
def run_tidy(task_queue, lock, timeout, failed_files): |
| 53 |
watchdog = None |
| 54 |
while True: |
| 55 |
command = task_queue.get() |
| 56 |
try: |
| 57 |
proc = subprocess.Popen( |
| 58 |
command, stdout=subprocess.PIPE, stderr=subprocess.PIPE |
| 59 |
) |
| 60 |
|
| 61 |
if timeout is not None: |
| 62 |
watchdog = threading.Timer(timeout, proc.kill) |
| 63 |
watchdog.start() |
| 64 |
|
| 65 |
stdout, stderr = proc.communicate() |
| 66 |
if proc.returncode != 0: |
| 67 |
if proc.returncode < 0: |
| 68 |
msg = "Terminated by signal %d : %s\n" % ( |
| 69 |
-proc.returncode, |
| 70 |
" ".join(command), |
| 71 |
) |
| 72 |
stderr += msg.encode("utf-8") |
| 73 |
failed_files.append(command) |
| 74 |
|
| 75 |
with lock: |
| 76 |
sys.stdout.write(stdout.decode("utf-8") + "\n") |
| 77 |
sys.stdout.flush() |
| 78 |
if stderr: |
| 79 |
sys.stderr.write(stderr.decode("utf-8") + "\n") |
| 80 |
sys.stderr.flush() |
| 81 |
except Exception as e: |
| 82 |
with lock: |
| 83 |
sys.stderr.write("Failed: " + str(e) + ": ".join(command) + "\n") |
| 84 |
finally: |
| 85 |
with lock: |
| 86 |
if not (timeout is None or watchdog is None): |
| 87 |
if not watchdog.is_alive(): |
| 88 |
sys.stderr.write( |
| 89 |
"Terminated by timeout: " + " ".join(command) + "\n" |
| 90 |
) |
| 91 |
watchdog.cancel() |
| 92 |
task_queue.task_done() |
| 93 |
|
| 94 |
|
| 95 |
def start_workers(max_tasks, tidy_caller, arguments): |
| 96 |
for _ in range(max_tasks): |
| 97 |
t = threading.Thread(target=tidy_caller, args=arguments) |
| 98 |
t.daemon = True |
| 99 |
t.start() |
| 100 |
|
| 101 |
|
| 102 |
def merge_replacement_files(tmpdir, mergefile): |
| 103 |
"""Merge all replacement files in a directory into a single file""" |
| 104 |
# The fixes suggested by clang-tidy >= 4.0.0 are given under |
| 105 |
# the top level key 'Diagnostics' in the output yaml files |
| 106 |
mergekey = "Diagnostics" |
| 107 |
merged = [] |
| 108 |
for replacefile in glob.iglob(os.path.join(tmpdir, "*.yaml")): |
| 109 |
content = yaml.safe_load(open(replacefile, "r")) |
| 110 |
if not content: |
| 111 |
continue # Skip empty files. |
| 112 |
merged.extend(content.get(mergekey, [])) |
| 113 |
|
| 114 |
if merged: |
| 115 |
# MainSourceFile: The key is required by the definition inside |
| 116 |
# include/clang/Tooling/ReplacementsYaml.h, but the value |
| 117 |
# is actually never used inside clang-apply-replacements, |
| 118 |
# so we set it to '' here. |
| 119 |
output = {"MainSourceFile": "", mergekey: merged} |
| 120 |
with open(mergefile, "w") as out: |
| 121 |
yaml.safe_dump(output, out) |
| 122 |
else: |
| 123 |
# Empty the file: |
| 124 |
open(mergefile, "w").close() |
| 125 |
|
| 126 |
|
| 127 |
def main(): |
| 128 |
parser = argparse.ArgumentParser( |
| 129 |
description="Run clang-tidy against changed files, and " |
| 130 |
"output diagnostics only for modified " |
| 131 |
"lines." |
| 132 |
) |
| 133 |
parser.add_argument( |
| 134 |
"-clang-tidy-binary", |
| 135 |
metavar="PATH", |
| 136 |
default="clang-tidy", |
| 137 |
help="path to clang-tidy binary", |
| 138 |
) |
| 139 |
parser.add_argument( |
| 140 |
"-p", |
| 141 |
metavar="NUM", |
| 142 |
default=0, |
| 143 |
help="strip the smallest prefix containing P slashes", |
| 144 |
) |
| 145 |
parser.add_argument( |
| 146 |
"-regex", |
| 147 |
metavar="PATTERN", |
| 148 |
default=None, |
| 149 |
help="custom pattern selecting file paths to check " |
| 150 |
"(case sensitive, overrides -iregex)", |
| 151 |
) |
| 152 |
parser.add_argument( |
| 153 |
"-iregex", |
| 154 |
metavar="PATTERN", |
| 155 |
default=r".*\.(cpp|cc|c\+\+|cxx|c|cl|h|hpp|m|mm|inc)", |
| 156 |
help="custom pattern selecting file paths to check " |
| 157 |
"(case insensitive, overridden by -regex)", |
| 158 |
) |
| 159 |
parser.add_argument( |
| 160 |
"-j", |
| 161 |
type=int, |
| 162 |
default=1, |
| 163 |
help="number of tidy instances to be run in parallel.", |
| 164 |
) |
| 165 |
parser.add_argument( |
| 166 |
"-timeout", type=int, default=None, help="timeout per each file in seconds." |
| 167 |
) |
| 168 |
parser.add_argument( |
| 169 |
"-fix", action="store_true", default=False, help="apply suggested fixes" |
| 170 |
) |
| 171 |
parser.add_argument( |
| 172 |
"-checks", |
| 173 |
help="checks filter, when not specified, use clang-tidy " "default", |
| 174 |
default="", |
| 175 |
) |
| 176 |
parser.add_argument( |
| 177 |
"-config-file", |
| 178 |
dest="config_file", |
| 179 |
help="Specify the path of .clang-tidy or custom config file", |
| 180 |
default="", |
| 181 |
) |
| 182 |
parser.add_argument("-use-color", action="store_true", help="Use colors in output") |
| 183 |
parser.add_argument( |
| 184 |
"-path", dest="build_path", help="Path used to read a compile command database." |
| 185 |
) |
| 186 |
if yaml: |
| 187 |
parser.add_argument( |
| 188 |
"-export-fixes", |
| 189 |
metavar="FILE_OR_DIRECTORY", |
| 190 |
dest="export_fixes", |
| 191 |
help="A directory or a yaml file to store suggested fixes in, " |
| 192 |
"which can be applied with clang-apply-replacements. If the " |
| 193 |
"parameter is a directory, the fixes of each compilation unit are " |
| 194 |
"stored in individual yaml files in the directory.", |
| 195 |
) |
| 196 |
else: |
| 197 |
parser.add_argument( |
| 198 |
"-export-fixes", |
| 199 |
metavar="DIRECTORY", |
| 200 |
dest="export_fixes", |
| 201 |
help="A directory to store suggested fixes in, which can be applied " |
| 202 |
"with clang-apply-replacements. The fixes of each compilation unit are " |
| 203 |
"stored in individual yaml files in the directory.", |
| 204 |
) |
| 205 |
parser.add_argument( |
| 206 |
"-extra-arg", |
| 207 |
dest="extra_arg", |
| 208 |
action="append", |
| 209 |
default=[], |
| 210 |
help="Additional argument to append to the compiler " "command line.", |
| 211 |
) |
| 212 |
parser.add_argument( |
| 213 |
"-extra-arg-before", |
| 214 |
dest="extra_arg_before", |
| 215 |
action="append", |
| 216 |
default=[], |
| 217 |
help="Additional argument to prepend to the compiler " "command line.", |
| 218 |
) |
| 219 |
parser.add_argument( |
| 220 |
"-quiet", |
| 221 |
action="store_true", |
| 222 |
default=False, |
| 223 |
help="Run clang-tidy in quiet mode", |
| 224 |
) |
| 225 |
parser.add_argument( |
| 226 |
"-load", |
| 227 |
dest="plugins", |
| 228 |
action="append", |
| 229 |
default=[], |
| 230 |
help="Load the specified plugin in clang-tidy.", |
| 231 |
) |
| 232 |
parser.add_argument( |
| 233 |
"-allow-no-checks", |
| 234 |
action="store_true", |
| 235 |
help="Allow empty enabled checks.", |
| 236 |
) |
| 237 |
|
| 238 |
clang_tidy_args = [] |
| 239 |
argv = sys.argv[1:] |
| 240 |
if "--" in argv: |
| 241 |
clang_tidy_args.extend(argv[argv.index("--") :]) |
| 242 |
argv = argv[: argv.index("--")] |
| 243 |
|
| 244 |
args = parser.parse_args(argv) |
| 245 |
|
| 246 |
# Extract changed lines for each file. |
| 247 |
filename = None |
| 248 |
lines_by_file = {} |
| 249 |
for line in sys.stdin: |
| 250 |
match = re.search('^\\+\\+\\+\\ "?(.*?/){%s}([^ \t\n"]*)' % args.p, line) |
| 251 |
if match: |
| 252 |
filename = match.group(2) |
| 253 |
if filename is None: |
| 254 |
continue |
| 255 |
|
| 256 |
if args.regex is not None: |
| 257 |
if not re.match("^%s$" % args.regex, filename): |
| 258 |
continue |
| 259 |
else: |
| 260 |
if not re.match("^%s$" % args.iregex, filename, re.IGNORECASE): |
| 261 |
continue |
| 262 |
|
| 263 |
match = re.search(r"^@@.*\+(\d+)(,(\d+))?", line) |
| 264 |
if match: |
| 265 |
start_line = int(match.group(1)) |
| 266 |
line_count = 1 |
| 267 |
if match.group(3): |
| 268 |
line_count = int(match.group(3)) |
| 269 |
if line_count == 0: |
| 270 |
continue |
| 271 |
end_line = start_line + line_count - 1 |
| 272 |
lines_by_file.setdefault(filename, []).append([start_line, end_line]) |
| 273 |
|
| 274 |
if not any(lines_by_file): |
| 275 |
print("No relevant changes found.") |
| 276 |
sys.exit(0) |
| 277 |
|
| 278 |
max_task_count = args.j |
| 279 |
if max_task_count == 0: |
| 280 |
max_task_count = multiprocessing.cpu_count() |
| 281 |
max_task_count = min(len(lines_by_file), max_task_count) |
| 282 |
|
| 283 |
combine_fixes = False |
| 284 |
export_fixes_dir = None |
| 285 |
delete_fixes_dir = False |
| 286 |
if args.export_fixes is not None: |
| 287 |
# if a directory is given, create it if it does not exist |
| 288 |
if args.export_fixes.endswith(os.path.sep) and not os.path.isdir( |
| 289 |
args.export_fixes |
| 290 |
): |
| 291 |
os.makedirs(args.export_fixes) |
| 292 |
|
| 293 |
if not os.path.isdir(args.export_fixes): |
| 294 |
if not yaml: |
| 295 |
raise RuntimeError( |
| 296 |
"Cannot combine fixes in one yaml file. Either install PyYAML or specify an output directory." |
| 297 |
) |
| 298 |
|
| 299 |
combine_fixes = True |
| 300 |
|
| 301 |
if os.path.isdir(args.export_fixes): |
| 302 |
export_fixes_dir = args.export_fixes |
| 303 |
|
| 304 |
if combine_fixes: |
| 305 |
export_fixes_dir = tempfile.mkdtemp() |
| 306 |
delete_fixes_dir = True |
| 307 |
|
| 308 |
# Tasks for clang-tidy. |
| 309 |
task_queue = queue.Queue(max_task_count) |
| 310 |
# A lock for console output. |
| 311 |
lock = threading.Lock() |
| 312 |
|
| 313 |
# List of files with a non-zero return code. |
| 314 |
failed_files = [] |
| 315 |
|
| 316 |
# Run a pool of clang-tidy workers. |
| 317 |
start_workers( |
| 318 |
max_task_count, run_tidy, (task_queue, lock, args.timeout, failed_files) |
| 319 |
) |
| 320 |
|
| 321 |
# Form the common args list. |
| 322 |
common_clang_tidy_args = [] |
| 323 |
if args.fix: |
| 324 |
common_clang_tidy_args.append("-fix") |
| 325 |
if args.checks != "": |
| 326 |
common_clang_tidy_args.append("-checks=" + args.checks) |
| 327 |
if args.config_file != "": |
| 328 |
common_clang_tidy_args.append("-config-file=" + args.config_file) |
| 329 |
if args.quiet: |
| 330 |
common_clang_tidy_args.append("-quiet") |
| 331 |
if args.build_path is not None: |
| 332 |
common_clang_tidy_args.append("-p=%s" % args.build_path) |
| 333 |
if args.use_color: |
| 334 |
common_clang_tidy_args.append("--use-color") |
| 335 |
if args.allow_no_checks: |
| 336 |
common_clang_tidy_args.append("--allow-no-checks") |
| 337 |
for arg in args.extra_arg: |
| 338 |
common_clang_tidy_args.append("-extra-arg=%s" % arg) |
| 339 |
for arg in args.extra_arg_before: |
| 340 |
common_clang_tidy_args.append("-extra-arg-before=%s" % arg) |
| 341 |
for plugin in args.plugins: |
| 342 |
common_clang_tidy_args.append("-load=%s" % plugin) |
| 343 |
|
| 344 |
for name in lines_by_file: |
| 345 |
line_filter_json = json.dumps( |
| 346 |
[{"name": name, "lines": lines_by_file[name]}], separators=(",", ":") |
| 347 |
) |
| 348 |
|
| 349 |
# Run clang-tidy on files containing changes. |
| 350 |
command = [args.clang_tidy_binary] |
| 351 |
command.append("-line-filter=" + line_filter_json) |
| 352 |
if args.export_fixes is not None: |
| 353 |
# Get a temporary file. We immediately close the handle so clang-tidy can |
| 354 |
# overwrite it. |
| 355 |
(handle, tmp_name) = tempfile.mkstemp(suffix=".yaml", dir=export_fixes_dir) |
| 356 |
os.close(handle) |
| 357 |
command.append("-export-fixes=" + tmp_name) |
| 358 |
command.extend(common_clang_tidy_args) |
| 359 |
command.append(name) |
| 360 |
command.extend(clang_tidy_args) |
| 361 |
|
| 362 |
task_queue.put(command) |
| 363 |
|
| 364 |
# Application return code |
| 365 |
return_code = 0 |
| 366 |
|
| 367 |
# Wait for all threads to be done. |
| 368 |
task_queue.join() |
| 369 |
# Application return code |
| 370 |
return_code = 0 |
| 371 |
if failed_files: |
| 372 |
return_code = 1 |
| 373 |
|
| 374 |
if combine_fixes: |
| 375 |
print("Writing fixes to " + args.export_fixes + " ...") |
| 376 |
try: |
| 377 |
merge_replacement_files(export_fixes_dir, args.export_fixes) |
| 378 |
except: |
| 379 |
sys.stderr.write("Error exporting fixes.\n") |
| 380 |
traceback.print_exc() |
| 381 |
return_code = 1 |
| 382 |
|
| 383 |
if delete_fixes_dir: |
| 384 |
shutil.rmtree(export_fixes_dir) |
| 385 |
sys.exit(return_code) |
| 386 |
|
| 387 |
|
| 388 |
if __name__ == "__main__": |
| 389 |
main() |