Debian Codegraph @

copyrights

raw ◀ ▶
1 Debian patch touch this file
1 #!/usr/bin/python3
2
•3 # A trivial script for updating copyright headers within a project from its git
4 # commit history
5 #
6 # Copyright (c) 2020-2021 Dave Jones <[email protected]>
7 #
8 # SPDX-License-Identifier: BSD-3-Clause
9
10 """
11 A script for updating the copyright headers on all files project-wide. It
12 derives the authorship and copyright years information from the git history
13 of the project; hence, this script must be run within a git repository.
14 """
15
16 from __future__ import annotations
17
18 import os
19 import sys
20 assert sys.version_info >= (3, 6), 'Script requires Python 3.6+'
21 import tempfile
22 import typing as t
23 from argparse import ArgumentParser, Namespace
24 from configparser import ConfigParser
25 from operator import attrgetter
26 from itertools import groupby
27 from datetime import datetime
28 from subprocess import Popen, PIPE, DEVNULL
29 from pathlib import Path
30 from fnmatch import fnmatch
31
32
33 SPDX_PREFIX = 'SPDX-License-Identifier:'
34 COPYRIGHT_PREFIX = 'Copyright (c)'
35
36
37 def main(args: t.List[str] = None):
38 if args is None:
39 args = sys.argv[1:]
40 config = get_config(args)
41
42 writer = CopyWriter.from_config(config)
43 for path, copyrights in get_copyrights(config.include, config.exclude):
44 print(f'Re-writing {path}...')
45 copyrights = sorted(
46 copyrights, reverse=True, key=lambda c: (max(c.years), c.author))
47 with AtomicReplaceFile(path, encoding='utf-8') as target:
48 with path.open('r') as source:
49 for chunk in writer.transform(source, copyrights):
50 target.write(chunk)
51
52
53 def get_config(args: t.List[str]) -> Namespace:
54 config = ConfigParser(
55 defaults={
56 'include': '**/*',
57 'exclude': '',
58 'license': 'LICENSE.txt',
59 'preamble': '',
60 'strip_preamble': 'false',
61 'spdx_prefix': SPDX_PREFIX,
62 'copy_prefix': COPYRIGHT_PREFIX,
63 },
64 delimiters=('=',), default_section='copyrights:settings',
65 empty_lines_in_values=False, interpolation=None,
66 converters={'list': lambda s: s.strip().splitlines() })
67 config.read('setup.cfg')
68 sect = config[config.default_section]
69
70 parser = ArgumentParser(description=__doc__)
71 parser.add_argument(
72 '-i', '--include', action='append', metavar='GLOB',
73 default=sect.getlist('include'),
74 help="The set of patterns that a file must match to be included in "
75 "the set of files to re-write. Can be specified multiple times to "
76 "add several patterns. Default: %(default)r")
77 parser.add_argument(
78 '-e', '--exclude', action='append', metavar='GLOB',
79 default=sect.getlist('exclude'),
80 help="The set of patterns that a file must *not* match to be included "
81 "in the set of files to re-write. Can be specified multiple times to "
82 "add several patterns. Default: %(default)r")
83 parser.add_argument(
84 '-l', '--license', action='store', type=Path, metavar='PATH',
85 default=sect['license'],
86 help="The file containing the project's license text. If this file "
87 "contains a SPDX-License-Identifier line (in addition to the license "
88 "text itself), then matching license text found in source files will "
89 "be replaced by the SPDX-License-Identifier line (appropriately "
90 "commented). Default: %(default)s")
91 parser.add_argument(
92 '-p', '--preamble', action='append', metavar='STR',
93 default=sect.getlist('preamble'),
94 help="The line(s) of text to insert before the copyright attributions "
95 "in source files. This is typically a brief description of the "
96 "project. Can be specified multiple times to add several lines. "
97 "Default: %(default)r")
98 parser.add_argument(
99 '-S', '--spdx-prefix', action='store', metavar='STR',
100 default=sect['spdx_prefix'],
101 help="The prefix on the line in the license file, and within comments "
102 "of source files that identifies the appropriate license from the "
103 "SPDX list. Default: %(default)r")
104 parser.add_argument(
105 '-C', '--copy-prefix', action='store', metavar='STR',
106 default=sect['copy_prefix'],
107 help="The prefix before copyright attributions in source files. "
108 "Default: %(default)r")
109 parser.add_argument(
110 '--no-strip-preamble', action='store_false', dest='strip_preamble')
111 parser.add_argument(
112 '--strip-preamble', action='store_true',
113 default=sect.getboolean('strip-preamble'),
114 help="If enabled, any existing preamble matching that specified "
115 "by --preamble will be removed. This can be used to change the "
116 "preamble text in files by first specifying the old preamble with "
117 "this option, then running a second time with the new preamble")
118
119 ns = parser.parse_args(args)
120 ns.include = set(ns.include)
121 ns.exclude = set(ns.exclude)
122 return ns
123
124
125 class Copyright(t.NamedTuple):
126 author: str
127 email: str
128 years: t.Set[int]
129
130 def __str__(self):
131 if len(self.years) > 1:
132 years = f'{min(self.years)}-{max(self.years)}'
133 else:
134 years = f'{min(self.years)}'
135 return f'{years} {self.author} <{self.email}>'
136
137
138 def get_copyrights(include: t.Set[str], exclude: t.Set[str])\
139 -> t.Iterator[t.Tuple[Path, t.Container[Copyright]]]:
140 sorted_blame = sorted(
141 get_contributions(include, exclude),
142 key=lambda c: (c.path, c.author, c.email)
143 )
144 blame_by_file = {
145 path: list(file_contributions)
146 for path, file_contributions in groupby(
147 sorted_blame, key=attrgetter('path')
148 )
149 }
150 for path, file_contributors in blame_by_file.items():
151 it = groupby(file_contributors, key=lambda c: (c.author, c.email))
152 copyrights = [
153 Copyright(author, email, {y.year for y in years})
154 for (author, email), years in it
155 ]
156 yield path, copyrights
157
158
159 class Contribution(t.NamedTuple):
160 author: str
161 email: str
162 year: int
163 path: Path
164
165
166 def get_contributions(include: t.Set[str], exclude: t.Set[str])\
167 -> t.Iterator[Contribution]:
168 for path in get_source_paths(include, exclude):
169 blame = Popen(
170 ['git', 'blame', '--line-porcelain', 'HEAD', '--', str(path)],
171 stdout=PIPE,
172 stderr=PIPE,
173 universal_newlines=True
174 )
175 author = email = year = None
176 if blame.stdout is not None:
177 for line in blame.stdout:
178 if line.startswith('author '):
179 author = line.split(' ', 1)[1].rstrip()
180 elif line.startswith('author-mail '):
181 email = line.split(' ', 1)[1].rstrip()
182 email = email.lstrip('<').rstrip('>')
183 elif line.startswith('author-time '):
184 # Forget the timezone; we only want the year anyway
185 timestamp = int(line.split(' ', 1)[1].strip())
186 year = datetime.fromtimestamp(timestamp).year
187 elif line.startswith('filename '):
188 assert author is not None
189 assert email is not None
190 assert year is not None
191 yield Contribution(
192 author=author, email=email, year=year, path=path)
193 author = email = year = None
194 blame.wait()
195 assert blame.returncode == 0
196
197
198 def get_source_paths(include: t.Set[str], exclude: t.Set[str])\
199 -> t.Iterator[Path]:
200 ls_tree = Popen(
201 ['git', 'ls-tree', '-r', '--name-only', 'HEAD'],
202 stdout=PIPE, stderr=DEVNULL, universal_newlines=True)
203 if not include:
204 include = {'*'}
205 if ls_tree.stdout is not None:
206 for filename in ls_tree.stdout:
207 filename = filename.strip()
208 if any(fnmatch(filename, pattern) for pattern in exclude):
209 continue
210 if any(fnmatch(filename, pattern) for pattern in include):
211 yield Path(filename)
212 ls_tree.wait()
213 assert ls_tree.returncode == 0
214
215
216 class License(t.NamedTuple):
217 ident: t.Optional[str]
218 text: t.List[str]
219
220
221 def get_license(path: Path, *, spdx_prefix: str = SPDX_PREFIX) -> License:
222 with open(path, 'r') as f:
223 lines = f.read().splitlines()
224
225 idents = [
226 line.rstrip() for line in lines
227 if line.startswith(spdx_prefix)
228 ]
229 ident = None
230 if len(idents) > 1:
231 raise RuntimeError(f'More than one {spdx_prefix} line in {path}!')
232 elif len(idents) == 1:
233 ident = idents[0]
234
235 body = [
236 line.rstrip() for line in lines
237 if not line.startswith(spdx_prefix)
238 ]
239 while not body[0]:
240 del body[0]
241 while not body[-1]:
242 del body[-1]
243 return License(ident, body)
244
245
246 class CopyWriter:
247 """
248 Transformer for the copyright header in source files. The :meth:`transform`
249 method can be called with a file-like object as the *source* and will
250 yield chunks of replacement data to be written to the replacement.
251 """
252
253 # The script's kinda dumb at this point - only handles straight-forward
254 # line-based comments, not multi-line delimited styles like /*..*/
255 COMMENTS = {
256 '': '#',
257 '.c': '//',
258 '.cpp': '//',
259 '.js': '//',
260 '.py': '#',
261 '.rst': '..',
262 '.sh': '#',
263 '.sql': '--',
264 }
265
266 def __init__(self, license: Path=Path('LICENSE.txt'),
267 preamble: t.List[str]=None,
268 spdx_prefix: str=SPDX_PREFIX,
269 copy_prefix: str=COPYRIGHT_PREFIX):
270 if preamble is None:
271 preamble = []
272 self.license = get_license(license, spdx_prefix=spdx_prefix)
273 self.preamble = preamble
274 self.spdx_prefix = spdx_prefix
275 self.copy_prefix = copy_prefix
276
277 @classmethod
278 def from_config(cls, config: Namespace) -> CopyWriter:
279 return cls(
280 config.license, config.preamble,
281 config.spdx_prefix, config.copy_prefix)
282
283 def transform(self, source: t.TextIO,
284 copyrights: t.List[Copyright], *,
285 comment_prefix: str=None) -> t.Iterator[str]:
286 if comment_prefix is None:
287 comment_prefix = self.COMMENTS[Path(source.name).suffix]
288 license_start = self.license.text[0]
289 license_end = self.license.text[-1]
290 state = 'header'
291 empty = True
292 for linenum, line in enumerate(source, start=1):
293 if state == 'header':
294 if linenum == 1 and line.startswith('#!'):
295 yield line
296 empty = False
297 elif linenum < 3 and (
298 'fileencoding=' in line or '-*- coding:' in line):
299 yield line
300 empty = False
301 elif line.rstrip() == comment_prefix:
302 pass # skip blank comment lines
303 elif line.startswith(f'{comment_prefix} {self.spdx_prefix}'):
304 pass # skip existing SPDX ident
305 elif line.startswith(f'{comment_prefix} {self.copy_prefix}'):
306 pass # skip existing copyright lines
307 elif any(line.startswith(f'{comment_prefix} {pre_line}')
308 for pre_line in self.preamble):
309 pass # skip existing preamble
310 elif line.startswith(f'{comment_prefix} {license_start}'):
311 state = 'license' # skip existing license lines
312 else:
313 yield from self._generate_header(
314 copyrights, comment_prefix, empty)
315 state = 'blank'
316 elif state == 'license':
317 if line.startswith(f'{comment_prefix} {license_end}'):
318 yield from self._generate_header(
319 copyrights, comment_prefix, empty)
320 state = 'blank'
321 continue
322 if state == 'blank':
323 # Ensure there's a blank line between license and start of the
324 # source body
325 if line.strip():
326 yield '\n'
327 yield line
328 state = 'body'
329 elif state == 'body':
330 yield line
331
332 def _generate_header(self, copyrights: t.Iterable[Copyright],
333 comment_prefix: str, empty: bool) -> t.Iterator[str]:
334 if not empty:
335 yield comment_prefix + '\n'
336 for line in self.preamble:
337 yield f'{comment_prefix} {line}\n'
338 if self.preamble:
339 yield comment_prefix + '\n'
340 for copyright in copyrights:
341 yield f'{comment_prefix} {self.copy_prefix} {copyright!s}\n'
342 yield comment_prefix + '\n'
343 if self.license.ident:
344 yield f'{comment_prefix} {self.license.ident}\n'
345 else:
346 for line in self.license.text:
347 if line:
348 yield f'{comment_prefix} {line}\n'
349 else:
350 yield comment_prefix + '\n'
351
352
353 class AtomicReplaceFile:
354 """
355 A context manager for atomically replacing a target file.
356
357 Uses :class:`tempfile.NamedTemporaryFile` to construct a temporary file in
358 the same directory as the target file. The associated file-like object is
359 returned as the context manager's variable; you should write the content
360 you wish to this object.
361
362 When the context manager exits, if no exception has occurred, the temporary
363 file will be renamed over the target file atomically (after copying
364 permissions from the target file). If an exception occurs during the
365 context manager's block, the temporary file will be deleted leaving the
366 original target file unaffected and the exception will be re-raised.
367
368 :param pathlib.Path path:
369 The full path and filename of the target file. This is expected to be
370 an absolute path.
371
372 :param str encoding:
373 If ``None`` (the default), the temporary file will be opened in binary
374 mode. Otherwise, this specifies the encoding to use with text mode.
375 """
376 def __init__(self, path: t.Union[str, Path], encoding: str = None):
377 if isinstance(path, str):
378 path = Path(path)
379 self._path = path
380 self._tempfile = tempfile.NamedTemporaryFile(
381 mode='wb' if encoding is None else 'w',
382 dir=str(self._path.parent), encoding=encoding, delete=False)
383 self._withfile = None
384
385 def __enter__(self):
386 self._withfile = self._tempfile.__enter__()
387 return self._withfile
388
389 def __exit__(self, exc_type, exc_value, exc_tb):
390 os.fchmod(self._withfile.file.fileno(), self._path.stat().st_mode)
391 result = self._tempfile.__exit__(exc_type, exc_value, exc_tb)
392 if exc_type is None:
393 os.rename(self._withfile.name, str(self._path))
394 else:
395 os.unlink(self._withfile.name)
396 return result
397
398
399 if __name__ == '__main__':
400 main()