| 1 |
#!/usr/bin/env python |
| 2 |
# |
| 3 |
# Copyright (c) 2014 Luis R. Rodriguez <[email protected]> |
| 4 |
# Copyright (c) 2013 Johannes Berg <[email protected]> |
| 5 |
# |
| 6 |
# This file is released under the GPLv2. |
| 7 |
# |
| 8 |
# Python wrapper for Coccinelle for multithreaded support, |
| 9 |
# designed to be used for working on a git tree, and with sensible |
| 10 |
# defaults, specifically for kernel developers. |
| 11 |
|
| 12 |
from multiprocessing import Process, cpu_count, Queue |
| 13 |
import argparse, subprocess, os, sys, re |
| 14 |
import tempfile, shutil |
| 15 |
import uuid |
| 16 |
|
| 17 |
class ReqError(Exception): |
| 18 |
pass |
| 19 |
class ExecutionError(ReqError): |
| 20 |
def __init__(self, errcode): |
| 21 |
self.error_code = errcode |
| 22 |
|
| 23 |
class Req: |
| 24 |
"To be used for verifying binary package dependencies on Python code" |
| 25 |
def __init__(self, chatty=True): |
| 26 |
self.all_reqs_ok = True |
| 27 |
self.debug = False |
| 28 |
self.chatty = chatty |
| 29 |
def logwrite(self, msg): |
| 30 |
if self.chatty: |
| 31 |
sys.stdout.write(msg) |
| 32 |
sys.stdout.flush() |
| 33 |
def enable_debug(self): |
| 34 |
self.debug = True |
| 35 |
def reqs_match(self): |
| 36 |
if self.all_reqs_ok: |
| 37 |
return True |
| 38 |
self.logwrite("You have unfulfilled binary requirements\n") |
| 39 |
return False |
| 40 |
def req_missing(self, program): |
| 41 |
self.all_reqs_ok = False |
| 42 |
self.logwrite("You need to have installed: %s\n" % program) |
| 43 |
def req_old_program(self, program, version_req): |
| 44 |
self.all_reqs_ok = False |
| 45 |
self.logwrite("You need to have installed: %s >= %s\n" % (program, version_req)) |
| 46 |
def which(self, program): |
| 47 |
cmd = ['which', program] |
| 48 |
process = subprocess.Popen(cmd, |
| 49 |
stdout=subprocess.PIPE, stderr=subprocess.STDOUT, |
| 50 |
close_fds=True, universal_newlines=True) |
| 51 |
stdout = process.communicate()[0] |
| 52 |
process.wait() |
| 53 |
if process.returncode != 0: |
| 54 |
raise ExecutionError(process.returncode) |
| 55 |
return stdout |
| 56 |
def req_exists(self, program): |
| 57 |
cmd = ['which', program] |
| 58 |
process = subprocess.Popen(cmd, |
| 59 |
stdout=subprocess.PIPE, stderr=subprocess.STDOUT, |
| 60 |
close_fds=True, universal_newlines=True) |
| 61 |
stdout = process.communicate()[0] |
| 62 |
process.wait() |
| 63 |
if process.returncode == 0: |
| 64 |
return True |
| 65 |
return False |
| 66 |
def req_get_prog_version(self, program, version_query, version_pos): |
| 67 |
''' |
| 68 |
Suppose you have a binary that outputs: |
| 69 |
$ spatch --version |
| 70 |
spatch version 1.0.0-rc21 with Python support and with PCRE support |
| 71 |
|
| 72 |
Every program veries what it wants you to query it for a version string, |
| 73 |
prog_version() is designed so that you pass what the program expects for |
| 74 |
its version query, and the position you expect the version string to be |
| 75 |
on using python list. |
| 76 |
''' |
| 77 |
cmd = [program, version_query] |
| 78 |
process = subprocess.Popen(cmd, |
| 79 |
stdout=subprocess.PIPE, stderr=subprocess.STDOUT, |
| 80 |
close_fds=True, universal_newlines=True) |
| 81 |
stdout = process.communicate()[0] |
| 82 |
process.wait() |
| 83 |
if process.returncode != 0: |
| 84 |
raise ExecutionError(process.returncode) |
| 85 |
if self.debug: |
| 86 |
sys.stdout.write("Running '%s' got us this break down:\n%s\n" % |
| 87 |
( |
| 88 |
' '.join(cmd), |
| 89 |
"\n".join(map(str, [[i, x] for i, x in enumerate(stdout.split())])), |
| 90 |
)) |
| 91 |
sys.stdout.write("You are using for version: %s\n" % stdout.split()[version_pos]) |
| 92 |
sys.stdout.write("Specifically your idx, element: %s\n" % ([[i, x] for i, x in enumerate(stdout.split())][version_pos])) |
| 93 |
return stdout.split()[version_pos] |
| 94 |
|
| 95 |
MAX_RC = 25 |
| 96 |
def __compute_rel_weight(self, rel_specs): |
| 97 |
weight = 0 |
| 98 |
extra = 0 |
| 99 |
sublevel = 0 |
| 100 |
relmod = 0 |
| 101 |
|
| 102 |
if self.debug: |
| 103 |
sys.stdout.write("VERSION = %s\n" % rel_specs['VERSION']) |
| 104 |
sys.stdout.write("PATCHLEVEL = %s\n" % rel_specs['PATCHLEVEL']) |
| 105 |
sys.stdout.write("SUBLEVEL = %s\n" % rel_specs['SUBLEVEL']) |
| 106 |
sys.stdout.write("EXTRAVERSION = %s\n" % rel_specs['EXTRAVERSION']) |
| 107 |
sys.stdout.write("RELMOD_UPDATE = %s\n" % rel_specs['RELMOD_UPDATE']) |
| 108 |
|
| 109 |
if rel_specs['EXTRAVERSION'] != '': |
| 110 |
if ("." in rel_specs['EXTRAVERSION'] or |
| 111 |
"rc" in rel_specs['EXTRAVERSION']): |
| 112 |
rc = rel_specs['EXTRAVERSION'].lstrip("-rc") |
| 113 |
if (rc == ""): |
| 114 |
rc = 0 |
| 115 |
else: |
| 116 |
rc = int(rc) - (Req.MAX_RC + 1) |
| 117 |
extra = int(rc) |
| 118 |
else: |
| 119 |
extra = int(rel_specs['EXTRAVERSION']) + 10 |
| 120 |
|
| 121 |
if rel_specs['SUBLEVEL'] != '': |
| 122 |
sublevel = int(rel_specs['SUBLEVEL'].lstrip(".")) * 20 |
| 123 |
else: |
| 124 |
sublevel = 5 |
| 125 |
|
| 126 |
if rel_specs['RELMOD_UPDATE'] != '': |
| 127 |
mod = rel_specs['RELMOD_UPDATE'] |
| 128 |
if (mod == ""): |
| 129 |
mod = 0 |
| 130 |
else: |
| 131 |
mod = int(mod) |
| 132 |
relmod = int(mod) |
| 133 |
|
| 134 |
weight = (int(rel_specs['VERSION']) << 32) + \ |
| 135 |
(int(rel_specs['PATCHLEVEL']) << 16) + \ |
| 136 |
(sublevel << 8 ) + \ |
| 137 |
(extra * 60) + (relmod * 2) |
| 138 |
|
| 139 |
return weight |
| 140 |
def req_get_rel_spec(self, rel): |
| 141 |
if "rc" in rel: |
| 142 |
m = re.match(r"v*(?P<VERSION>\d+)\.+" |
| 143 |
"(?P<PATCHLEVEL>\d+)[.]*" |
| 144 |
"(?P<SUBLEVEL>\d*)" |
| 145 |
"(?P<EXTRAVERSION>[-rc]+\w*)\-*" |
| 146 |
"(?P<RELMOD_UPDATE>\d*)[-]*", |
| 147 |
rel) |
| 148 |
else: |
| 149 |
m = re.match(r"v*(?P<VERSION>\d+)\.+" |
| 150 |
"(?P<PATCHLEVEL>\d+)[.]*" |
| 151 |
"(?P<SUBLEVEL>\d*)[.]*" |
| 152 |
"(?P<EXTRAVERSION>\w*)\-*" |
| 153 |
"(?P<RELMOD_UPDATE>\d*)[-]*", |
| 154 |
rel) |
| 155 |
if not m: |
| 156 |
return m |
| 157 |
rel_specs = m.groupdict() |
| 158 |
return rel_specs |
| 159 |
def compute_rel_weight(self, rel): |
| 160 |
rel_specs = self.req_get_rel_spec(rel) |
| 161 |
if not rel_specs: |
| 162 |
return 0 |
| 163 |
return self.__compute_rel_weight(rel_specs) |
| 164 |
def linux_version_cmp(self, version_req, version): |
| 165 |
''' |
| 166 |
If the program follows the linux version style scheme you can |
| 167 |
use this to compare versions. |
| 168 |
''' |
| 169 |
weight_has = self.compute_rel_weight(version) |
| 170 |
weight_req = self.compute_rel_weight(version_req) |
| 171 |
|
| 172 |
if self.debug: |
| 173 |
sys.stdout.write("You have program weight: %s\n" % weight_has) |
| 174 |
sys.stdout.write("Required program weight: %s\n" % weight_req) |
| 175 |
|
| 176 |
if weight_has < weight_req: |
| 177 |
return -1 |
| 178 |
return 0 |
| 179 |
def require_version(self, program, version_query, version_req, version_pos, version_cmp): |
| 180 |
''' |
| 181 |
If you have a program version requirement you can specify it here, |
| 182 |
as for the other flags refer to prog_version. |
| 183 |
''' |
| 184 |
if not self.require(program): |
| 185 |
return False |
| 186 |
version = self.req_get_prog_version(program, version_query, version_pos) |
| 187 |
if self.debug: |
| 188 |
sys.stdout.write("Checking release specs and weight: for: %s\n" % program) |
| 189 |
sys.stdout.write("You have version: %s\n" % version) |
| 190 |
sys.stdout.write("Required version: %s\n" % version_req) |
| 191 |
if version_cmp(version_req, version) != 0: |
| 192 |
self.req_old_program(program, version_req) |
| 193 |
return False |
| 194 |
return True |
| 195 |
def require(self, program): |
| 196 |
if self.req_exists(program): |
| 197 |
return True |
| 198 |
self.req_missing(program) |
| 199 |
return False |
| 200 |
def require_hint(self, program, package_hint): |
| 201 |
if self.require(program): |
| 202 |
return True |
| 203 |
sys.stdout.write("Try installing the package: %s\n" % package_hint) |
| 204 |
return False |
| 205 |
def coccinelle(self, version): |
| 206 |
if self.require_version('spatch', '--version', version, 2, self.linux_version_cmp): |
| 207 |
return True |
| 208 |
self.logwrite("Try installing the package: coccinelle\n") |
| 209 |
self.logwrite("If that is too old go grab the code from source:\n\n") |
| 210 |
self.logwrite("git clone https://github.com/coccinelle/coccinelle.git\n\n") |
| 211 |
self.logwrite("To build you will need: ocaml ncurses-devel\n\n") |
| 212 |
self.logwrite("If on SUSE / OpenSUSE you will also need: ocaml-ocamldoc\n\n") |
| 213 |
return False |
| 214 |
def make(self, version): |
| 215 |
return self.require_version('make', '--version', version, 2, self.linux_version_cmp) |
| 216 |
def gcc(self, version): |
| 217 |
return self.require_version('gcc', '--version', version, 3, self.linux_version_cmp) |
| 218 |
|
| 219 |
class GitError(Exception): |
| 220 |
pass |
| 221 |
class ExecutionGitError(GitError): |
| 222 |
def __init__(self, errcode): |
| 223 |
self.error_code = errcode |
| 224 |
|
| 225 |
def _check(process): |
| 226 |
if process.returncode != 0: |
| 227 |
raise ExecutionError(process.returncode) |
| 228 |
|
| 229 |
def git_init(tree=None): |
| 230 |
process = subprocess.Popen(['git', 'init'], |
| 231 |
stdout=subprocess.PIPE, stderr=subprocess.STDOUT, |
| 232 |
close_fds=True, universal_newlines=True, cwd=tree) |
| 233 |
stdout = process.communicate()[0] |
| 234 |
process.wait() |
| 235 |
_check(process) |
| 236 |
|
| 237 |
def git_rev_parse(tree=None, extra_args=None): |
| 238 |
cmd = ['git', 'rev-parse' ] + extra_args |
| 239 |
process = subprocess.Popen(cmd, |
| 240 |
stdout=subprocess.PIPE, stderr=subprocess.STDOUT, |
| 241 |
close_fds=True, universal_newlines=True, cwd=tree) |
| 242 |
stdout = process.communicate()[0] |
| 243 |
process.wait() |
| 244 |
if process.returncode != 0: |
| 245 |
return None |
| 246 |
return stdout.split('\n', 1)[0] |
| 247 |
|
| 248 |
def gitname(path=None): |
| 249 |
path = os.path.abspath(path) |
| 250 |
work_dir = path |
| 251 |
if not os.path.isdir(path): |
| 252 |
work_dir = os.path.abspath(os.path.dirname(path)) |
| 253 |
process = subprocess.Popen(['git', 'rev-parse', '--show-toplevel', path], |
| 254 |
stdout=subprocess.PIPE, stderr=subprocess.STDOUT, |
| 255 |
close_fds=True, universal_newlines=True, cwd=work_dir) |
| 256 |
stdout = process.communicate()[0] |
| 257 |
process.wait() |
| 258 |
if process.returncode != 0: |
| 259 |
return None |
| 260 |
return stdout.split('\n', 1)[0] |
| 261 |
|
| 262 |
def git_add(path, tree=None): |
| 263 |
process = subprocess.Popen(['git', 'add', '--ignore-removal', path], |
| 264 |
stdout=subprocess.PIPE, stderr=subprocess.STDOUT, |
| 265 |
close_fds=True, universal_newlines=True, cwd=tree) |
| 266 |
stdout = process.communicate()[0] |
| 267 |
process.wait() |
| 268 |
_check(process) |
| 269 |
|
| 270 |
def git_checkout(tree=None, extra_args=None): |
| 271 |
cmd = ['git', 'checkout' ] + extra_args |
| 272 |
process = subprocess.Popen(cmd, |
| 273 |
stdout=subprocess.PIPE, stderr=subprocess.STDOUT, |
| 274 |
close_fds=True, universal_newlines=True, cwd=tree) |
| 275 |
stdout = process.communicate()[0] |
| 276 |
process.wait() |
| 277 |
_check(process) |
| 278 |
|
| 279 |
def git_branch(tree=None, extra_args=None): |
| 280 |
cmd = ['git', 'branch' ] + extra_args |
| 281 |
process = subprocess.Popen(cmd, |
| 282 |
stdout=subprocess.PIPE, stderr=subprocess.STDOUT, |
| 283 |
close_fds=True, universal_newlines=True, cwd=tree) |
| 284 |
stdout = process.communicate()[0] |
| 285 |
process.wait() |
| 286 |
_check(process) |
| 287 |
|
| 288 |
def git_commit_all(message, tree=None): |
| 289 |
git_add('.', tree=tree) |
| 290 |
process = subprocess.Popen(['git', 'commit', '--allow-empty', '-a', '-m', message], |
| 291 |
stdout=subprocess.PIPE, stderr=subprocess.STDOUT, |
| 292 |
close_fds=True, universal_newlines=True, cwd=tree) |
| 293 |
stdout = process.communicate()[0] |
| 294 |
process.wait() |
| 295 |
_check(process) |
| 296 |
|
| 297 |
def git_diff(tree=None, extra_args=None): |
| 298 |
cmd = ['git', 'diff', '--color=always'] + extra_args |
| 299 |
|
| 300 |
process = subprocess.Popen(cmd, |
| 301 |
stdout=subprocess.PIPE, stderr=subprocess.STDOUT, |
| 302 |
close_fds=True, universal_newlines=True, cwd=tree) |
| 303 |
stdout = process.communicate()[0] |
| 304 |
process.wait() |
| 305 |
_check(process) |
| 306 |
|
| 307 |
return stdout |
| 308 |
|
| 309 |
# simple tempdir wrapper object for 'with' statement |
| 310 |
# |
| 311 |
# Usage: |
| 312 |
# with tempdir.tempdir() as tmpdir: |
| 313 |
# os.chdir(tmpdir) |
| 314 |
# do something |
| 315 |
# |
| 316 |
class tempdir(object): |
| 317 |
def __init__(self, suffix='', prefix='', dir=None, nodelete=False): |
| 318 |
self.suffix = '' |
| 319 |
self.prefix = '' |
| 320 |
self.dir = dir |
| 321 |
self.nodelete = nodelete |
| 322 |
|
| 323 |
def __enter__(self): |
| 324 |
self._name = tempfile.mkdtemp(suffix=self.suffix, |
| 325 |
prefix=self.prefix, |
| 326 |
dir=self.dir) |
| 327 |
return self._name |
| 328 |
|
| 329 |
def __exit__(self, type, value, traceback): |
| 330 |
if self.nodelete: |
| 331 |
print('not deleting directory %s!' % self._name) |
| 332 |
else: |
| 333 |
shutil.rmtree(self._name) |
| 334 |
|
| 335 |
def apply_patches(args, patch_src, target_dir, logwrite=lambda x:None): |
| 336 |
""" |
| 337 |
Given a path of a directories of patches apply the patches |
| 338 |
""" |
| 339 |
patches = [] |
| 340 |
for root, dirs, files in os.walk(patch_src): |
| 341 |
for f in files: |
| 342 |
if f.endswith('.patch'): |
| 343 |
patches.append(os.path.join(root, f)) |
| 344 |
patches.sort() |
| 345 |
prefix_len = len(patch_src) + 1 |
| 346 |
for pfile in patches: |
| 347 |
print_name = pfile[prefix_len:] |
| 348 |
|
| 349 |
if args.show_proof or args.verbose: |
| 350 |
logwrite("Applying patch %s\n" % pfile) |
| 351 |
|
| 352 |
process = subprocess.Popen(['patch', '-p1'], stdout=subprocess.PIPE, |
| 353 |
stderr=subprocess.STDOUT, stdin=subprocess.PIPE, |
| 354 |
close_fds=True, universal_newlines=True, |
| 355 |
cwd=target_dir) |
| 356 |
output = process.communicate(input=open(pfile, 'r').read())[0] |
| 357 |
output = output.split('\n') |
| 358 |
if output[-1] == '': |
| 359 |
output = output[:-1] |
| 360 |
if process.returncode != 0: |
| 361 |
if not args.verbose: |
| 362 |
logwrite("Failed to apply changes from %s" % print_name) |
| 363 |
for line in output: |
| 364 |
logwrite('> %s' % line) |
| 365 |
raise Exception('Patch failed') |
| 366 |
|
| 367 |
# remove orig/rej files that patch sometimes creates |
| 368 |
for root, dirs, files in os.walk(target_dir): |
| 369 |
for f in files: |
| 370 |
if f[-5:] == '.orig' or f[-4:] == '.rej': |
| 371 |
os.unlink(os.path.join(root, f)) |
| 372 |
git_commit_all(tree=target_dir, message="apply patch %s" % (print_name)) |
| 373 |
|
| 374 |
class CoccinelleError(Exception): |
| 375 |
pass |
| 376 |
|
| 377 |
class ExecutionErrorThread(CoccinelleError): |
| 378 |
def __init__(self, errcode, fn, cocci_file, threads, t, logwrite, print_name): |
| 379 |
self.error_code = errcode |
| 380 |
logwrite("Failed to apply changes from %s\n" % print_name) |
| 381 |
|
| 382 |
logwrite("Specific log output from change that failed using %s\n" % print_name) |
| 383 |
tf = open(fn, 'r') |
| 384 |
for line in tf.read(): |
| 385 |
logwrite(line) |
| 386 |
tf.close() |
| 387 |
|
| 388 |
logwrite("Full log using %s\n" % print_name) |
| 389 |
for num in range(threads): |
| 390 |
fn = os.path.join(t, '.tmp_spatch_worker.' + str(num)) |
| 391 |
if (not os.path.isfile(fn)): |
| 392 |
continue |
| 393 |
tf = open(fn, 'r') |
| 394 |
for line in tf.read(): |
| 395 |
logwrite(line) |
| 396 |
tf.close() |
| 397 |
os.unlink(fn) |
| 398 |
|
| 399 |
class ExecutionErrorCocci(CoccinelleError): |
| 400 |
def __init__(self, errcode, output, cocci_file, logwrite, print_name): |
| 401 |
self.error_code = errcode |
| 402 |
logwrite("Failed to apply changes from %s\n" % print_name) |
| 403 |
logwrite(output) |
| 404 |
|
| 405 |
def spatch(cocci_file, outdir, logwrite, num_jobs, print_name, extra_args=[], |
| 406 |
verbose=False): |
| 407 |
|
| 408 |
req = Req(chatty=True) |
| 409 |
req.coccinelle('1.0.2') |
| 410 |
|
| 411 |
if not req.reqs_match(): |
| 412 |
sys.exit(1) |
| 413 |
|
| 414 |
num_cpus = cpu_count() |
| 415 |
if num_jobs: |
| 416 |
threads = int(num_jobs) |
| 417 |
else: |
| 418 |
threads = num_cpus |
| 419 |
|
| 420 |
cmd = ['spatch', |
| 421 |
'--sp-file', cocci_file, |
| 422 |
'--in-place', |
| 423 |
'--recursive-includes', |
| 424 |
'--relax-include-path', |
| 425 |
'--timeout', '120', |
| 426 |
'--dir', outdir ] |
| 427 |
|
| 428 |
if (threads > 1): |
| 429 |
cmd.extend(['--jobs', str(threads)]) |
| 430 |
|
| 431 |
cmd.extend(extra_args) |
| 432 |
|
| 433 |
if verbose: |
| 434 |
logwrite("%s\n" % " ".join(cmd)) |
| 435 |
|
| 436 |
sprocess = subprocess.Popen(cmd, |
| 437 |
stdout=subprocess.PIPE, stderr=subprocess.STDOUT, |
| 438 |
close_fds=True, universal_newlines=True) |
| 439 |
output = sprocess.communicate()[0] |
| 440 |
sprocess.wait() |
| 441 |
if sprocess.returncode != 0: |
| 442 |
raise ExecutionErrorCocci(sprocess.returncode, output, cocci_file, logwrite, print_name) |
| 443 |
return output |
| 444 |
|
| 445 |
def spatch_old(cocci_file, outdir, |
| 446 |
max_threads, thread_id, temp_dir, ret_q, extra_args=[], |
| 447 |
verbose=False): |
| 448 |
cmd = ['spatch', |
| 449 |
'--sp-file', cocci_file, |
| 450 |
'--in-place', |
| 451 |
'--recursive-includes', |
| 452 |
'--relax-include-path', |
| 453 |
'--timeout', '120', |
| 454 |
'--dir', outdir ] |
| 455 |
|
| 456 |
if (max_threads > 1): |
| 457 |
cmd.extend(['-max', str(max_threads), '-index', str(thread_id)]) |
| 458 |
|
| 459 |
cmd.extend(extra_args) |
| 460 |
|
| 461 |
fn = os.path.join(temp_dir, '.tmp_spatch_worker.' + str(thread_id)) |
| 462 |
outfile = open(fn, 'w') |
| 463 |
if verbose: |
| 464 |
logwrite("%s\n" % " ".join(cmd)) |
| 465 |
|
| 466 |
sprocess = subprocess.Popen(cmd, |
| 467 |
stdout=outfile, stderr=subprocess.STDOUT, |
| 468 |
close_fds=True, universal_newlines=True) |
| 469 |
sprocess.wait() |
| 470 |
outfile.close() |
| 471 |
ret_q.put((sprocess.returncode, fn)) |
| 472 |
|
| 473 |
def threaded_spatch(cocci_file, outdir, logwrite, num_jobs, |
| 474 |
print_name, extra_args=[], verbose=True): |
| 475 |
num_cpus = cpu_count() |
| 476 |
if num_jobs: |
| 477 |
threads = int(num_jobs) |
| 478 |
else: |
| 479 |
threads = num_cpus |
| 480 |
jobs = list() |
| 481 |
output = "" |
| 482 |
ret_q = Queue() |
| 483 |
with tempdir() as t: |
| 484 |
for num in range(threads): |
| 485 |
p = Process(target=spatch_old, args=(cocci_file, outdir, |
| 486 |
threads, num, t, ret_q, |
| 487 |
extra_args)) |
| 488 |
jobs.append(p) |
| 489 |
for p in jobs: |
| 490 |
p.start() |
| 491 |
|
| 492 |
for num in range(threads): |
| 493 |
ret, fn = ret_q.get() |
| 494 |
if ret != 0: |
| 495 |
raise ExecutionErrorThread(ret, fn, cocci_file, threads, t, |
| 496 |
logwrite, print_name) |
| 497 |
for job in jobs: |
| 498 |
p.join() |
| 499 |
|
| 500 |
for num in range(threads): |
| 501 |
fn = os.path.join(t, '.tmp_spatch_worker.' + str(num)) |
| 502 |
tf = open(fn, 'r') |
| 503 |
output = output + tf.read() |
| 504 |
tf.close() |
| 505 |
os.unlink(fn) |
| 506 |
return output |
| 507 |
|
| 508 |
def logwrite(msg): |
| 509 |
sys.stdout.write(msg) |
| 510 |
sys.stdout.flush() |
| 511 |
|
| 512 |
def _main(): |
| 513 |
parser = argparse.ArgumentParser(description='Wrapper around Coccinelle spatch ' + |
| 514 |
'which infers which options to enable.') |
| 515 |
parser.add_argument('cocci_file', metavar='<SmPL patch>', type=str, |
| 516 |
help='This is the Coccinelle file you want to use') |
| 517 |
parser.add_argument('target_dir', metavar='<target>', type=str, |
| 518 |
help='Target directory or file to modify') |
| 519 |
parser.add_argument('-p', '--profile-cocci', const=True, default=False, action="store_const", |
| 520 |
help='Enable profile, this will pass --profile to Coccinelle.') |
| 521 |
parser.add_argument('-s', '--show-proof', const=True, default=False, action="store_const", |
| 522 |
help='Show proof that the provided SmPL patch can replace a respective patch series') |
| 523 |
parser.add_argument('-c', '--clean-proof', const=True, default=False, action="store_const", |
| 524 |
help='Show proof that the provided SmPL patch can replace a respective patch series, and' + |
| 525 |
'if the proof shows equivalence, delete any temporary branches used to test the proof.' + |
| 526 |
'If any errors were encountered the branches used to test the proof will be kept to ' + |
| 527 |
'enable debugging') |
| 528 |
parser.add_argument('-j', '--jobs', metavar='<jobs>', type=str, default=None, |
| 529 |
help='Only use the cocci file passed for Coccinelle, don\'t do anything else, ' + |
| 530 |
'also creates a git repo on the target directory for easy inspection ' + |
| 531 |
'of changes done by Coccinelle.') |
| 532 |
parser.add_argument('-v', '--verbose', const=True, default=False, action="store_const", |
| 533 |
help='Enable output from Coccinelle') |
| 534 |
args = parser.parse_args() |
| 535 |
|
| 536 |
if not os.path.isfile(args.cocci_file): |
| 537 |
return -2 |
| 538 |
if not os.path.isfile(args.target_dir) and not os.path.isdir(args.target_dir): |
| 539 |
logwrite("Path (%s) is not a file or directory\n" % (args.target_dir)) |
| 540 |
return -2 |
| 541 |
|
| 542 |
current_branch = None |
| 543 |
smpl_branch_name = "pycocci-smpl-" + str(uuid.uuid4())[:8] |
| 544 |
patch_branch_name = "pycocci-patch-" + str(uuid.uuid4())[:8] |
| 545 |
|
| 546 |
extra_spatch_args = [] |
| 547 |
if args.profile_cocci: |
| 548 |
extra_spatch_args.append('--profile') |
| 549 |
jobs = 0 |
| 550 |
if args.jobs > 0: |
| 551 |
jobs = args.jobs |
| 552 |
|
| 553 |
has_spatch_1_0_1 = Req(chatty=False) |
| 554 |
has_spatch_1_0_1.coccinelle('1.0.1') |
| 555 |
|
| 556 |
has_spatch_1_0_2 = Req(chatty=False) |
| 557 |
has_spatch_1_0_2.coccinelle('1.0.2') |
| 558 |
|
| 559 |
git_reqs = Req(chatty=False) |
| 560 |
git_reqs.require('git') |
| 561 |
|
| 562 |
glimpse_index = os.path.abspath(os.path.join(args.target_dir, '.glimpse_index')) |
| 563 |
git_dir = None |
| 564 |
|
| 565 |
if git_reqs.reqs_match(): |
| 566 |
git_dir = gitname(path=args.target_dir) |
| 567 |
|
| 568 |
if args.show_proof or args.clean_proof: |
| 569 |
# As an example if you use --show-proof patches/collateral-evolutions/network/09-threaded-irq.cocci |
| 570 |
# the patches under 09-threaded-irq will be used for the proof. |
| 571 |
patch_src = args.cocci_file.split('/')[-1].split('.cocci')[0] |
| 572 |
dirname = os.path.dirname(args.cocci_file) |
| 573 |
patch_src = os.path.abspath(os.path.join(dirname, patch_src)) |
| 574 |
if not os.path.isdir(patch_src): |
| 575 |
logwrite("Path given (%s) must be a directory with patches\n" % (patch_src)) |
| 576 |
return -2 |
| 577 |
git_reqs = Req(chatty=True) |
| 578 |
git_reqs.require('git') |
| 579 |
if not git_dir: |
| 580 |
if os.path.isfile(args.target_dir): |
| 581 |
logwrite("Path given (%s) is a file, try passing the directory " |
| 582 |
"(%s) if you are certain you want us to create a git repo to provide a" |
| 583 |
"a proof there\n" % (args.target_dir, os.path.dirname(args.target_dir))) |
| 584 |
return -2 |
| 585 |
logwrite("Path (%s) not part of a git tree, creating one for you...\n" % (args.target_dir)) |
| 586 |
git_init(tree=args.target_dir) |
| 587 |
git_commit_all(tree=args.target_dir, message="Initial commit") |
| 588 |
else: |
| 589 |
diff_stat = git_diff(tree=args.target_dir, extra_args = [ '--stat' ] ) |
| 590 |
if len(diff_stat) != 0: |
| 591 |
logwrite("SmPL <--> Patch proof equivalence requires a " |
| 592 |
"clean tree.\nYou have a dirty tree with " |
| 593 |
"uncommitted changes:\n\n") |
| 594 |
logwrite(diff_stat) |
| 595 |
logwrite("\nEither commit these changes, or stash them away somewhere\n") |
| 596 |
return -2 |
| 597 |
cmd = [ '--abbrev-ref', 'HEAD' ] |
| 598 |
current_branch = git_rev_parse(tree=args.target_dir, extra_args = cmd) |
| 599 |
if args.show_proof or args.verbose: |
| 600 |
logwrite("\n") |
| 601 |
logwrite("Current branch: %s\n" % (current_branch)) |
| 602 |
logwrite("Patch branch: %s\n" % (patch_branch_name)) |
| 603 |
logwrite("SmPL branch: %s\n" % (smpl_branch_name)) |
| 604 |
logwrite("\n") |
| 605 |
git_checkout(tree=args.target_dir, extra_args = ['-b', smpl_branch_name]) |
| 606 |
git_checkout(tree=args.target_dir, extra_args = ['-b', patch_branch_name]) |
| 607 |
|
| 608 |
apply_patches(args, patch_src, args.target_dir, logwrite) |
| 609 |
|
| 610 |
git_checkout(tree=args.target_dir, extra_args = [smpl_branch_name]) |
| 611 |
|
| 612 |
if os.path.isfile(glimpse_index): |
| 613 |
extra_spatch_args.append('--use-glimpse') |
| 614 |
elif has_spatch_1_0_2.reqs_match(): |
| 615 |
if git_dir: |
| 616 |
extra_spatch_args.append('--use-gitgrep') |
| 617 |
else: |
| 618 |
extra_spatch_args.append('--use-coccigrep') |
| 619 |
else: |
| 620 |
extra_spatch_args.append('--use-coccigrep') |
| 621 |
|
| 622 |
show_spatch_cmd = True |
| 623 |
if args.clean_proof and not args.verbose: |
| 624 |
show_spatch_cmd = False |
| 625 |
if has_spatch_1_0_2.reqs_match(): |
| 626 |
output = spatch(args.cocci_file, args.target_dir, logwrite, jobs, |
| 627 |
os.path.basename(args.cocci_file), |
| 628 |
extra_args=extra_spatch_args, verbose=show_spatch_cmd) |
| 629 |
else: |
| 630 |
output = threaded_spatch(args.cocci_file, |
| 631 |
args.target_dir, |
| 632 |
logwrite, |
| 633 |
jobs, |
| 634 |
os.path.basename(args.cocci_file), |
| 635 |
extra_args=extra_spatch_args, |
| 636 |
verbose=show_spatch_cmd) |
| 637 |
if args.verbose: |
| 638 |
logwrite(output) |
| 639 |
if args.show_proof or args.clean_proof: |
| 640 |
git_commit_all(tree=args.target_dir, message="Initial commit") |
| 641 |
git_checkout(tree=args.target_dir, extra_args = [current_branch]) |
| 642 |
cmd = [ '--stat', patch_branch_name + ".." + smpl_branch_name ] |
| 643 |
diff_stat = git_diff(tree=args.target_dir, extra_args = cmd) |
| 644 |
if len(diff_stat) == 0: |
| 645 |
if args.show_proof or args.verbose: |
| 646 |
logwrite('\nSmPL patch fully replaces patch series!\n') |
| 647 |
else: |
| 648 |
logwrite('%s\t:OK\n' % args.cocci_file) |
| 649 |
if args.clean_proof: |
| 650 |
if args.verbose: |
| 651 |
logwrite('\nRequested clean proof, so deleting work branches\n') |
| 652 |
git_branch(tree=args.target_dir, extra_args = ['-D', patch_branch_name]) |
| 653 |
git_branch(tree=args.target_dir, extra_args = ['-D', smpl_branch_name]) |
| 654 |
else: |
| 655 |
if args.show_proof or args.verbose: |
| 656 |
logwrite('\nDifferences found:\n\n') |
| 657 |
logwrite('Change directory to %s and run:\n\n\tgit diff %s..%s\n\n' % (args.target_dir, patch_branch_name, smpl_branch_name)) |
| 658 |
logwrite('diffstat of the changes:\n') |
| 659 |
logwrite(diff_stat) |
| 660 |
else: |
| 661 |
logwrite('%s\t:FAIL\n' % args.cocci_file) |
| 662 |
return 0 |
| 663 |
|
| 664 |
if __name__ == '__main__': |
| 665 |
ret = _main() |
| 666 |
if ret: |
| 667 |
sys.exit(ret) |