Debian Codegraph @

Tools/patchcheck/patchcheck.py

raw
1 #!/usr/bin/env python3
2 """Check proposed changes for common issues."""
3 import re
4 import sys
5 import shutil
6 import os.path
7 import subprocess
8 import sysconfig
9
10 import reindent
11 import untabify
12
13
14 def get_python_source_dir():
15 src_dir = sysconfig.get_config_var('abs_srcdir')
16 if not src_dir:
17 src_dir = sysconfig.get_config_var('srcdir')
18 return os.path.abspath(src_dir)
19
20
21 # Excluded directories which are copies of external libraries:
22 # don't check their coding style
23 EXCLUDE_DIRS = [
24 os.path.join('Modules', '_decimal', 'libmpdec'),
25 os.path.join('Modules', 'expat'),
26 os.path.join('Modules', 'zlib'),
27 ]
28 SRCDIR = get_python_source_dir()
29
30
31 def n_files_str(count):
32 """Return 'N file(s)' with the proper plurality on 'file'."""
33 s = "s" if count != 1 else ""
34 return f"{count} file{s}"
35
36
37 def status(message, modal=False, info=None):
38 """Decorator to output status info to stdout."""
39 def decorated_fxn(fxn):
40 def call_fxn(*args, **kwargs):
41 sys.stdout.write(message + ' ... ')
42 sys.stdout.flush()
43 result = fxn(*args, **kwargs)
44 if not modal and not info:
45 print("done")
46 elif info:
47 print(info(result))
48 else:
49 print("yes" if result else "NO")
50 return result
51 return call_fxn
52 return decorated_fxn
53
54
55 def get_git_branch():
56 """Get the symbolic name for the current git branch"""
57 cmd = "git rev-parse --abbrev-ref HEAD".split()
58 try:
59 return subprocess.check_output(cmd,
60 stderr=subprocess.DEVNULL,
61 cwd=SRCDIR,
62 encoding='UTF-8')
63 except subprocess.CalledProcessError:
64 return None
65
66
67 def get_git_upstream_remote():
68 """
69 Get the remote name to use for upstream branches
70
71 Check for presence of "https://github.com/python/cpython" remote URL.
72 If only one is found, return that remote name. If multiple are found,
73 check for and return "upstream", "origin", or "python", in that
74 order. Raise an error if no valid matches are found.
75 """
76 cmd = "git remote -v".split()
77 output = subprocess.check_output(
78 cmd,
79 stderr=subprocess.DEVNULL,
80 cwd=SRCDIR,
81 encoding="UTF-8"
82 )
83 # Filter to desired remotes, accounting for potential uppercasing
84 filtered_remotes = {
85 remote.split("\t")[0].lower() for remote in output.split('\n')
86 if "python/cpython" in remote.lower() and remote.endswith("(fetch)")
87 }
88 if len(filtered_remotes) == 1:
89 [remote] = filtered_remotes
90 return remote
91 for remote_name in ["upstream", "origin", "python"]:
92 if remote_name in filtered_remotes:
93 return remote_name
94 remotes_found = "\n".join(
95 {remote for remote in output.split('\n') if remote.endswith("(fetch)")}
96 )
97 raise ValueError(
98 f"Patchcheck was unable to find an unambiguous upstream remote, "
99 f"with URL matching 'https://github.com/python/cpython'. "
100 f"For help creating an upstream remote, see Dev Guide: "
101 f"https://devguide.python.org/getting-started/"
102 f"git-boot-camp/#cloning-a-forked-cpython-repository "
103 f"\nRemotes found: \n{remotes_found}"
104 )
105
106
107 def get_git_remote_default_branch(remote_name):
108 """Get the name of the default branch for the given remote
109
110 It is typically called 'main', but may differ
111 """
112 cmd = f"git remote show {remote_name}".split()
113 env = os.environ.copy()
114 env['LANG'] = 'C'
115 try:
116 remote_info = subprocess.check_output(cmd,
117 stderr=subprocess.DEVNULL,
118 cwd=SRCDIR,
119 encoding='UTF-8',
120 env=env)
121 except subprocess.CalledProcessError:
122 return None
123 for line in remote_info.splitlines():
124 if "HEAD branch:" in line:
125 base_branch = line.split(":")[1].strip()
126 return base_branch
127 return None
128
129
130 @status("Getting base branch for PR",
131 info=lambda x: x if x is not None else "not a PR branch")
132 def get_base_branch():
133 if not os.path.exists(os.path.join(SRCDIR, '.git')):
134 # Not a git checkout, so there's no base branch
135 return None
136 upstream_remote = get_git_upstream_remote()
137 version = sys.version_info
138 if version.releaselevel == 'alpha':
139 base_branch = get_git_remote_default_branch(upstream_remote)
140 else:
141 base_branch = "{0.major}.{0.minor}".format(version)
142 this_branch = get_git_branch()
143 if this_branch is None or this_branch == base_branch:
144 # Not on a git PR branch, so there's no base branch
145 return None
146 return upstream_remote + "/" + base_branch
147
148
149 @status("Getting the list of files that have been added/changed",
150 info=lambda x: n_files_str(len(x)))
151 def changed_files(base_branch=None):
152 """Get the list of changed or added files from git."""
153 if os.path.exists(os.path.join(SRCDIR, '.git')):
154 # We just use an existence check here as:
155 # directory = normal git checkout/clone
156 # file = git worktree directory
157 if base_branch:
158 cmd = 'git diff --name-status ' + base_branch
159 else:
160 cmd = 'git status --porcelain'
161 filenames = []
162 with subprocess.Popen(cmd.split(),
163 stdout=subprocess.PIPE,
164 cwd=SRCDIR) as st:
165 git_file_status, _ = st.communicate()
166 if st.returncode != 0:
167 sys.exit(f'error running {cmd}')
168 for line in git_file_status.splitlines():
169 line = line.decode().rstrip()
170 status_text, filename = line.split(maxsplit=1)
171 status = set(status_text)
172 # modified, added or unmerged files
173 if not status.intersection('MAU'):
174 continue
175 if ' -> ' in filename:
176 # file is renamed
177 filename = filename.split(' -> ', 2)[1].strip()
178 filenames.append(filename)
179 else:
180 sys.exit('need a git checkout to get modified files')
181
182 filenames2 = []
183 for filename in filenames:
184 # Normalize the path to be able to match using .startswith()
185 filename = os.path.normpath(filename)
186 if any(filename.startswith(path) for path in EXCLUDE_DIRS):
187 # Exclude the file
188 continue
189 filenames2.append(filename)
190
191 return filenames2
192
193
194 def report_modified_files(file_paths):
195 count = len(file_paths)
196 if count == 0:
197 return n_files_str(count)
198 else:
199 lines = [f"{n_files_str(count)}:"]
200 for path in file_paths:
201 lines.append(f" {path}")
202 return "\n".join(lines)
203
204
205 #: Python files that have tabs by design:
206 _PYTHON_FILES_WITH_TABS = frozenset({
207 'Tools/c-analyzer/cpython/_parser.py',
208 })
209
210
211 @status("Fixing Python file whitespace", info=report_modified_files)
212 def normalize_whitespace(file_paths):
213 """Make sure that the whitespace for .py files have been normalized."""
214 reindent.makebackup = False # No need to create backups.
215 fixed = [
216 path for path in file_paths
217 if (
218 path.endswith('.py')
219 and path not in _PYTHON_FILES_WITH_TABS
220 and reindent.check(os.path.join(SRCDIR, path))
221 )
222 ]
223 return fixed
224
225
226 @status("Fixing C file whitespace", info=report_modified_files)
227 def normalize_c_whitespace(file_paths):
228 """Report if any C files """
229 fixed = []
230 for path in file_paths:
231 abspath = os.path.join(SRCDIR, path)
232 with open(abspath, 'r') as f:
233 if '\t' not in f.read():
234 continue
235 untabify.process(abspath, 8, verbose=False)
236 fixed.append(path)
237 return fixed
238
239
240 @status("Docs modified", modal=True)
241 def docs_modified(file_paths):
242 """Report if any file in the Doc directory has been changed."""
243 return bool(file_paths)
244
245
246 @status("Misc/ACKS updated", modal=True)
247 def credit_given(file_paths):
248 """Check if Misc/ACKS has been changed."""
249 return os.path.join('Misc', 'ACKS') in file_paths
250
251
252 @status("Misc/NEWS.d updated with `blurb`", modal=True)
253 def reported_news(file_paths):
254 """Check if Misc/NEWS.d has been changed."""
255 return any(p.startswith(os.path.join('Misc', 'NEWS.d', 'next'))
256 for p in file_paths)
257
258
259 @status("configure regenerated", modal=True, info=str)
260 def regenerated_configure(file_paths):
261 """Check if configure has been regenerated."""
262 if 'configure.ac' in file_paths:
263 return "yes" if 'configure' in file_paths else "no"
264 else:
265 return "not needed"
266
267
268 @status("pyconfig.h.in regenerated", modal=True, info=str)
269 def regenerated_pyconfig_h_in(file_paths):
270 """Check if pyconfig.h.in has been regenerated."""
271 if 'configure.ac' in file_paths:
272 return "yes" if 'pyconfig.h.in' in file_paths else "no"
273 else:
274 return "not needed"
275
276
277 def ci(pull_request):
278 if pull_request == 'false':
279 print('Not a pull request; skipping')
280 return
281 base_branch = get_base_branch()
282 file_paths = changed_files(base_branch)
283 python_files = [fn for fn in file_paths if fn.endswith('.py')]
284 c_files = [fn for fn in file_paths if fn.endswith(('.c', '.h'))]
285 fixed = []
286 fixed.extend(normalize_whitespace(python_files))
287 fixed.extend(normalize_c_whitespace(c_files))
288 if not fixed:
289 print('No whitespace issues found')
290 else:
291 count = len(fixed)
292 print(f'Please fix the {n_files_str(count)} with whitespace issues')
293 print('(on Unix you can run `make patchcheck` to make the fixes)')
294 sys.exit(1)
295
296
297 def main():
298 base_branch = get_base_branch()
299 file_paths = changed_files(base_branch)
300 python_files = [fn for fn in file_paths if fn.endswith('.py')]
301 c_files = [fn for fn in file_paths if fn.endswith(('.c', '.h'))]
302 doc_files = [fn for fn in file_paths if fn.startswith('Doc') and
303 fn.endswith(('.rst', '.inc'))]
304 misc_files = {p for p in file_paths if p.startswith('Misc')}
305 # PEP 8 whitespace rules enforcement.
306 normalize_whitespace(python_files)
307 # C rules enforcement.
308 normalize_c_whitespace(c_files)
309 # Docs updated.
310 docs_modified(doc_files)
311 # Misc/ACKS changed.
312 credit_given(misc_files)
313 # Misc/NEWS changed.
314 reported_news(misc_files)
315 # Regenerated configure, if necessary.
316 regenerated_configure(file_paths)
317 # Regenerated pyconfig.h.in, if necessary.
318 regenerated_pyconfig_h_in(file_paths)
319
320 # Test suite run and passed.
321 if python_files or c_files:
322 end = " and check for refleaks?" if c_files else "?"
323 print()
324 print("Did you run the test suite" + end)
325
326
327 if __name__ == '__main__':
328 import argparse
329 parser = argparse.ArgumentParser(description=__doc__)
330 parser.add_argument('--ci',
331 help='Perform pass/fail checks')
332 args = parser.parse_args()
333 if args.ci:
334 ci(args.ci)
335 else:
336 main()