| 1 |
#!/usr/bin/env python3 |
| 2 |
"""Check proposed changes for common issues.""" |
| 3 |
import re |
| 4 |
import sys |
| 5 |
import shutil |
| 6 |
import os.path |
| 7 |
import subprocess |
| 8 |
import sysconfig |
| 9 |
|
| 10 |
import reindent |
| 11 |
import untabify |
| 12 |
|
| 13 |
|
| 14 |
def get_python_source_dir(): |
| 15 |
src_dir = sysconfig.get_config_var('abs_srcdir') |
| 16 |
if not src_dir: |
| 17 |
src_dir = sysconfig.get_config_var('srcdir') |
| 18 |
return os.path.abspath(src_dir) |
| 19 |
|
| 20 |
|
| 21 |
# Excluded directories which are copies of external libraries: |
| 22 |
# don't check their coding style |
| 23 |
EXCLUDE_DIRS = [ |
| 24 |
os.path.join('Modules', '_decimal', 'libmpdec'), |
| 25 |
os.path.join('Modules', 'expat'), |
| 26 |
os.path.join('Modules', 'zlib'), |
| 27 |
] |
| 28 |
SRCDIR = get_python_source_dir() |
| 29 |
|
| 30 |
|
| 31 |
def n_files_str(count): |
| 32 |
"""Return 'N file(s)' with the proper plurality on 'file'.""" |
| 33 |
s = "s" if count != 1 else "" |
| 34 |
return f"{count} file{s}" |
| 35 |
|
| 36 |
|
| 37 |
def status(message, modal=False, info=None): |
| 38 |
"""Decorator to output status info to stdout.""" |
| 39 |
def decorated_fxn(fxn): |
| 40 |
def call_fxn(*args, **kwargs): |
| 41 |
sys.stdout.write(message + ' ... ') |
| 42 |
sys.stdout.flush() |
| 43 |
result = fxn(*args, **kwargs) |
| 44 |
if not modal and not info: |
| 45 |
print("done") |
| 46 |
elif info: |
| 47 |
print(info(result)) |
| 48 |
else: |
| 49 |
print("yes" if result else "NO") |
| 50 |
return result |
| 51 |
return call_fxn |
| 52 |
return decorated_fxn |
| 53 |
|
| 54 |
|
| 55 |
def get_git_branch(): |
| 56 |
"""Get the symbolic name for the current git branch""" |
| 57 |
cmd = "git rev-parse --abbrev-ref HEAD".split() |
| 58 |
try: |
| 59 |
return subprocess.check_output(cmd, |
| 60 |
stderr=subprocess.DEVNULL, |
| 61 |
cwd=SRCDIR, |
| 62 |
encoding='UTF-8') |
| 63 |
except subprocess.CalledProcessError: |
| 64 |
return None |
| 65 |
|
| 66 |
|
| 67 |
def get_git_upstream_remote(): |
| 68 |
""" |
| 69 |
Get the remote name to use for upstream branches |
| 70 |
|
| 71 |
Check for presence of "https://github.com/python/cpython" remote URL. |
| 72 |
If only one is found, return that remote name. If multiple are found, |
| 73 |
check for and return "upstream", "origin", or "python", in that |
| 74 |
order. Raise an error if no valid matches are found. |
| 75 |
""" |
| 76 |
cmd = "git remote -v".split() |
| 77 |
output = subprocess.check_output( |
| 78 |
cmd, |
| 79 |
stderr=subprocess.DEVNULL, |
| 80 |
cwd=SRCDIR, |
| 81 |
encoding="UTF-8" |
| 82 |
) |
| 83 |
# Filter to desired remotes, accounting for potential uppercasing |
| 84 |
filtered_remotes = { |
| 85 |
remote.split("\t")[0].lower() for remote in output.split('\n') |
| 86 |
if "python/cpython" in remote.lower() and remote.endswith("(fetch)") |
| 87 |
} |
| 88 |
if len(filtered_remotes) == 1: |
| 89 |
[remote] = filtered_remotes |
| 90 |
return remote |
| 91 |
for remote_name in ["upstream", "origin", "python"]: |
| 92 |
if remote_name in filtered_remotes: |
| 93 |
return remote_name |
| 94 |
remotes_found = "\n".join( |
| 95 |
{remote for remote in output.split('\n') if remote.endswith("(fetch)")} |
| 96 |
) |
| 97 |
raise ValueError( |
| 98 |
f"Patchcheck was unable to find an unambiguous upstream remote, " |
| 99 |
f"with URL matching 'https://github.com/python/cpython'. " |
| 100 |
f"For help creating an upstream remote, see Dev Guide: " |
| 101 |
f"https://devguide.python.org/getting-started/" |
| 102 |
f"git-boot-camp/#cloning-a-forked-cpython-repository " |
| 103 |
f"\nRemotes found: \n{remotes_found}" |
| 104 |
) |
| 105 |
|
| 106 |
|
| 107 |
def get_git_remote_default_branch(remote_name): |
| 108 |
"""Get the name of the default branch for the given remote |
| 109 |
|
| 110 |
It is typically called 'main', but may differ |
| 111 |
""" |
| 112 |
cmd = f"git remote show {remote_name}".split() |
| 113 |
env = os.environ.copy() |
| 114 |
env['LANG'] = 'C' |
| 115 |
try: |
| 116 |
remote_info = subprocess.check_output(cmd, |
| 117 |
stderr=subprocess.DEVNULL, |
| 118 |
cwd=SRCDIR, |
| 119 |
encoding='UTF-8', |
| 120 |
env=env) |
| 121 |
except subprocess.CalledProcessError: |
| 122 |
return None |
| 123 |
for line in remote_info.splitlines(): |
| 124 |
if "HEAD branch:" in line: |
| 125 |
base_branch = line.split(":")[1].strip() |
| 126 |
return base_branch |
| 127 |
return None |
| 128 |
|
| 129 |
|
| 130 |
@status("Getting base branch for PR", |
| 131 |
info=lambda x: x if x is not None else "not a PR branch") |
| 132 |
def get_base_branch(): |
| 133 |
if not os.path.exists(os.path.join(SRCDIR, '.git')): |
| 134 |
# Not a git checkout, so there's no base branch |
| 135 |
return None |
| 136 |
upstream_remote = get_git_upstream_remote() |
| 137 |
version = sys.version_info |
| 138 |
if version.releaselevel == 'alpha': |
| 139 |
base_branch = get_git_remote_default_branch(upstream_remote) |
| 140 |
else: |
| 141 |
base_branch = "{0.major}.{0.minor}".format(version) |
| 142 |
this_branch = get_git_branch() |
| 143 |
if this_branch is None or this_branch == base_branch: |
| 144 |
# Not on a git PR branch, so there's no base branch |
| 145 |
return None |
| 146 |
return upstream_remote + "/" + base_branch |
| 147 |
|
| 148 |
|
| 149 |
@status("Getting the list of files that have been added/changed", |
| 150 |
info=lambda x: n_files_str(len(x))) |
| 151 |
def changed_files(base_branch=None): |
| 152 |
"""Get the list of changed or added files from git.""" |
| 153 |
if os.path.exists(os.path.join(SRCDIR, '.git')): |
| 154 |
# We just use an existence check here as: |
| 155 |
# directory = normal git checkout/clone |
| 156 |
# file = git worktree directory |
| 157 |
if base_branch: |
| 158 |
cmd = 'git diff --name-status ' + base_branch |
| 159 |
else: |
| 160 |
cmd = 'git status --porcelain' |
| 161 |
filenames = [] |
| 162 |
with subprocess.Popen(cmd.split(), |
| 163 |
stdout=subprocess.PIPE, |
| 164 |
cwd=SRCDIR) as st: |
| 165 |
git_file_status, _ = st.communicate() |
| 166 |
if st.returncode != 0: |
| 167 |
sys.exit(f'error running {cmd}') |
| 168 |
for line in git_file_status.splitlines(): |
| 169 |
line = line.decode().rstrip() |
| 170 |
status_text, filename = line.split(maxsplit=1) |
| 171 |
status = set(status_text) |
| 172 |
# modified, added or unmerged files |
| 173 |
if not status.intersection('MAU'): |
| 174 |
continue |
| 175 |
if ' -> ' in filename: |
| 176 |
# file is renamed |
| 177 |
filename = filename.split(' -> ', 2)[1].strip() |
| 178 |
filenames.append(filename) |
| 179 |
else: |
| 180 |
sys.exit('need a git checkout to get modified files') |
| 181 |
|
| 182 |
filenames2 = [] |
| 183 |
for filename in filenames: |
| 184 |
# Normalize the path to be able to match using .startswith() |
| 185 |
filename = os.path.normpath(filename) |
| 186 |
if any(filename.startswith(path) for path in EXCLUDE_DIRS): |
| 187 |
# Exclude the file |
| 188 |
continue |
| 189 |
filenames2.append(filename) |
| 190 |
|
| 191 |
return filenames2 |
| 192 |
|
| 193 |
|
| 194 |
def report_modified_files(file_paths): |
| 195 |
count = len(file_paths) |
| 196 |
if count == 0: |
| 197 |
return n_files_str(count) |
| 198 |
else: |
| 199 |
lines = [f"{n_files_str(count)}:"] |
| 200 |
for path in file_paths: |
| 201 |
lines.append(f" {path}") |
| 202 |
return "\n".join(lines) |
| 203 |
|
| 204 |
|
| 205 |
#: Python files that have tabs by design: |
| 206 |
_PYTHON_FILES_WITH_TABS = frozenset({ |
| 207 |
'Tools/c-analyzer/cpython/_parser.py', |
| 208 |
}) |
| 209 |
|
| 210 |
|
| 211 |
@status("Fixing Python file whitespace", info=report_modified_files) |
| 212 |
def normalize_whitespace(file_paths): |
| 213 |
"""Make sure that the whitespace for .py files have been normalized.""" |
| 214 |
reindent.makebackup = False # No need to create backups. |
| 215 |
fixed = [ |
| 216 |
path for path in file_paths |
| 217 |
if ( |
| 218 |
path.endswith('.py') |
| 219 |
and path not in _PYTHON_FILES_WITH_TABS |
| 220 |
and reindent.check(os.path.join(SRCDIR, path)) |
| 221 |
) |
| 222 |
] |
| 223 |
return fixed |
| 224 |
|
| 225 |
|
| 226 |
@status("Fixing C file whitespace", info=report_modified_files) |
| 227 |
def normalize_c_whitespace(file_paths): |
| 228 |
"""Report if any C files """ |
| 229 |
fixed = [] |
| 230 |
for path in file_paths: |
| 231 |
abspath = os.path.join(SRCDIR, path) |
| 232 |
with open(abspath, 'r') as f: |
| 233 |
if '\t' not in f.read(): |
| 234 |
continue |
| 235 |
untabify.process(abspath, 8, verbose=False) |
| 236 |
fixed.append(path) |
| 237 |
return fixed |
| 238 |
|
| 239 |
|
| 240 |
@status("Docs modified", modal=True) |
| 241 |
def docs_modified(file_paths): |
| 242 |
"""Report if any file in the Doc directory has been changed.""" |
| 243 |
return bool(file_paths) |
| 244 |
|
| 245 |
|
| 246 |
@status("Misc/ACKS updated", modal=True) |
| 247 |
def credit_given(file_paths): |
| 248 |
"""Check if Misc/ACKS has been changed.""" |
| 249 |
return os.path.join('Misc', 'ACKS') in file_paths |
| 250 |
|
| 251 |
|
| 252 |
@status("Misc/NEWS.d updated with `blurb`", modal=True) |
| 253 |
def reported_news(file_paths): |
| 254 |
"""Check if Misc/NEWS.d has been changed.""" |
| 255 |
return any(p.startswith(os.path.join('Misc', 'NEWS.d', 'next')) |
| 256 |
for p in file_paths) |
| 257 |
|
| 258 |
|
| 259 |
@status("configure regenerated", modal=True, info=str) |
| 260 |
def regenerated_configure(file_paths): |
| 261 |
"""Check if configure has been regenerated.""" |
| 262 |
if 'configure.ac' in file_paths: |
| 263 |
return "yes" if 'configure' in file_paths else "no" |
| 264 |
else: |
| 265 |
return "not needed" |
| 266 |
|
| 267 |
|
| 268 |
@status("pyconfig.h.in regenerated", modal=True, info=str) |
| 269 |
def regenerated_pyconfig_h_in(file_paths): |
| 270 |
"""Check if pyconfig.h.in has been regenerated.""" |
| 271 |
if 'configure.ac' in file_paths: |
| 272 |
return "yes" if 'pyconfig.h.in' in file_paths else "no" |
| 273 |
else: |
| 274 |
return "not needed" |
| 275 |
|
| 276 |
|
| 277 |
def ci(pull_request): |
| 278 |
if pull_request == 'false': |
| 279 |
print('Not a pull request; skipping') |
| 280 |
return |
| 281 |
base_branch = get_base_branch() |
| 282 |
file_paths = changed_files(base_branch) |
| 283 |
python_files = [fn for fn in file_paths if fn.endswith('.py')] |
| 284 |
c_files = [fn for fn in file_paths if fn.endswith(('.c', '.h'))] |
| 285 |
fixed = [] |
| 286 |
fixed.extend(normalize_whitespace(python_files)) |
| 287 |
fixed.extend(normalize_c_whitespace(c_files)) |
| 288 |
if not fixed: |
| 289 |
print('No whitespace issues found') |
| 290 |
else: |
| 291 |
count = len(fixed) |
| 292 |
print(f'Please fix the {n_files_str(count)} with whitespace issues') |
| 293 |
print('(on Unix you can run `make patchcheck` to make the fixes)') |
| 294 |
sys.exit(1) |
| 295 |
|
| 296 |
|
| 297 |
def main(): |
| 298 |
base_branch = get_base_branch() |
| 299 |
file_paths = changed_files(base_branch) |
| 300 |
python_files = [fn for fn in file_paths if fn.endswith('.py')] |
| 301 |
c_files = [fn for fn in file_paths if fn.endswith(('.c', '.h'))] |
| 302 |
doc_files = [fn for fn in file_paths if fn.startswith('Doc') and |
| 303 |
fn.endswith(('.rst', '.inc'))] |
| 304 |
misc_files = {p for p in file_paths if p.startswith('Misc')} |
| 305 |
# PEP 8 whitespace rules enforcement. |
| 306 |
normalize_whitespace(python_files) |
| 307 |
# C rules enforcement. |
| 308 |
normalize_c_whitespace(c_files) |
| 309 |
# Docs updated. |
| 310 |
docs_modified(doc_files) |
| 311 |
# Misc/ACKS changed. |
| 312 |
credit_given(misc_files) |
| 313 |
# Misc/NEWS changed. |
| 314 |
reported_news(misc_files) |
| 315 |
# Regenerated configure, if necessary. |
| 316 |
regenerated_configure(file_paths) |
| 317 |
# Regenerated pyconfig.h.in, if necessary. |
| 318 |
regenerated_pyconfig_h_in(file_paths) |
| 319 |
|
| 320 |
# Test suite run and passed. |
| 321 |
if python_files or c_files: |
| 322 |
end = " and check for refleaks?" if c_files else "?" |
| 323 |
print() |
| 324 |
print("Did you run the test suite" + end) |
| 325 |
|
| 326 |
|
| 327 |
if __name__ == '__main__': |
| 328 |
import argparse |
| 329 |
parser = argparse.ArgumentParser(description=__doc__) |
| 330 |
parser.add_argument('--ci', |
| 331 |
help='Perform pass/fail checks') |
| 332 |
args = parser.parse_args() |
| 333 |
if args.ci: |
| 334 |
ci(args.ci) |
| 335 |
else: |
| 336 |
main() |