Debian Codegraph @

copyrights

raw ◀ ▶
Source reconstructed at patch state 0 of 1 — not the installed (fully patched) source.
1 Debian patch touch this file
1 #!/usr/bin/python3
2
3 """
4 A script for updating the copyright headers on all files project-wide. It
5 derives the authorship and copyright years information from the git history
6 of the project; hence, this script must be run within a git repository.
7 """
8
9 from __future__ import annotations
10
11 import os
12 import sys
13 assert sys.version_info >= (3, 6), 'Script requires Python 3.6+'
14 import tempfile
15 import typing as t
16 from argparse import ArgumentParser, Namespace
17 from configparser import ConfigParser
18 from operator import attrgetter
19 from itertools import groupby
20 from datetime import datetime
21 from subprocess import Popen, PIPE, DEVNULL
22 from pathlib import Path
23 from fnmatch import fnmatch
24
25
26 SPDX_PREFIX = 'SPDX-License-Identifier:'
27 COPYRIGHT_PREFIX = 'Copyright (c)'
28
29
30 def main(args: t.List[str] = None):
31 if args is None:
32 args = sys.argv[1:]
33 config = get_config(args)
34
35 writer = CopyWriter.from_config(config)
36 for path, copyrights in get_copyrights(config.include, config.exclude):
37 print(f'Re-writing {path}...')
38 copyrights = sorted(
39 copyrights, reverse=True, key=lambda c: (max(c.years), c.author))
40 with AtomicReplaceFile(path, encoding='utf-8') as target:
41 with path.open('r') as source:
42 for chunk in writer.transform(source, copyrights):
43 target.write(chunk)
44
45
46 def get_config(args: t.List[str]) -> Namespace:
47 config = ConfigParser(
48 defaults={
49 'include': '**/*',
50 'exclude': '',
51 'license': 'LICENSE.txt',
52 'preamble': '',
53 'strip_preamble': 'false',
54 'spdx_prefix': SPDX_PREFIX,
55 'copy_prefix': COPYRIGHT_PREFIX,
56 },
57 delimiters=('=',), default_section='copyrights:settings',
58 empty_lines_in_values=False, interpolation=None,
59 converters={'list': lambda s: s.strip().splitlines() })
60 config.read('setup.cfg')
61 sect = config[config.default_section]
62
63 parser = ArgumentParser(description=__doc__)
64 parser.add_argument(
65 '-i', '--include', action='append', metavar='GLOB',
66 default=sect.getlist('include'),
67 help="The set of patterns that a file must match to be included in "
68 "the set of files to re-write. Can be specified multiple times to "
69 "add several patterns. Default: %(default)r")
70 parser.add_argument(
71 '-e', '--exclude', action='append', metavar='GLOB',
72 default=sect.getlist('exclude'),
73 help="The set of patterns that a file must *not* match to be included "
74 "in the set of files to re-write. Can be specified multiple times to "
75 "add several patterns. Default: %(default)r")
76 parser.add_argument(
77 '-l', '--license', action='store', type=Path, metavar='PATH',
78 default=sect['license'],
79 help="The file containing the project's license text. If this file "
80 "contains a SPDX-License-Identifier line (in addition to the license "
81 "text itself), then matching license text found in source files will "
82 "be replaced by the SPDX-License-Identifier line (appropriately "
83 "commented). Default: %(default)s")
84 parser.add_argument(
85 '-p', '--preamble', action='append', metavar='STR',
86 default=sect.getlist('preamble'),
87 help="The line(s) of text to insert before the copyright attributions "
88 "in source files. This is typically a brief description of the "
89 "project. Can be specified multiple times to add several lines. "
90 "Default: %(default)r")
91 parser.add_argument(
92 '-S', '--spdx-prefix', action='store', metavar='STR',
93 default=sect['spdx_prefix'],
94 help="The prefix on the line in the license file, and within comments "
95 "of source files that identifies the appropriate license from the "
96 "SPDX list. Default: %(default)r")
97 parser.add_argument(
98 '-C', '--copy-prefix', action='store', metavar='STR',
99 default=sect['copy_prefix'],
100 help="The prefix before copyright attributions in source files. "
101 "Default: %(default)r")
102 parser.add_argument(
103 '--no-strip-preamble', action='store_false', dest='strip_preamble')
104 parser.add_argument(
105 '--strip-preamble', action='store_true',
106 default=sect.getboolean('strip-preamble'),
107 help="If enabled, any existing preamble matching that specified "
108 "by --preamble will be removed. This can be used to change the "
109 "preamble text in files by first specifying the old preamble with "
110 "this option, then running a second time with the new preamble")
111
112 ns = parser.parse_args(args)
113 ns.include = set(ns.include)
114 ns.exclude = set(ns.exclude)
115 return ns
116
117
118 class Copyright(t.NamedTuple):
119 author: str
120 email: str
121 years: t.Set[int]
122
123 def __str__(self):
124 if len(self.years) > 1:
125 years = f'{min(self.years)}-{max(self.years)}'
126 else:
127 years = f'{min(self.years)}'
128 return f'{years} {self.author} <{self.email}>'
129
130
131 def get_copyrights(include: t.Set[str], exclude: t.Set[str])\
132 -> t.Iterator[t.Tuple[Path, t.Container[Copyright]]]:
133 sorted_blame = sorted(
134 get_contributions(include, exclude),
135 key=lambda c: (c.path, c.author, c.email)
136 )
137 blame_by_file = {
138 path: list(file_contributions)
139 for path, file_contributions in groupby(
140 sorted_blame, key=attrgetter('path')
141 )
142 }
143 for path, file_contributors in blame_by_file.items():
144 it = groupby(file_contributors, key=lambda c: (c.author, c.email))
145 copyrights = [
146 Copyright(author, email, {y.year for y in years})
147 for (author, email), years in it
148 ]
149 yield path, copyrights
150
151
152 class Contribution(t.NamedTuple):
153 author: str
154 email: str
155 year: int
156 path: Path
157
158
159 def get_contributions(include: t.Set[str], exclude: t.Set[str])\
160 -> t.Iterator[Contribution]:
161 for path in get_source_paths(include, exclude):
162 blame = Popen(
163 ['git', 'blame', '--line-porcelain', 'HEAD', '--', str(path)],
164 stdout=PIPE,
165 stderr=PIPE,
166 universal_newlines=True
167 )
168 author = email = year = None
169 if blame.stdout is not None:
170 for line in blame.stdout:
171 if line.startswith('author '):
172 author = line.split(' ', 1)[1].rstrip()
173 elif line.startswith('author-mail '):
174 email = line.split(' ', 1)[1].rstrip()
175 email = email.lstrip('<').rstrip('>')
176 elif line.startswith('author-time '):
177 # Forget the timezone; we only want the year anyway
178 timestamp = int(line.split(' ', 1)[1].strip())
179 year = datetime.fromtimestamp(timestamp).year
180 elif line.startswith('filename '):
181 assert author is not None
182 assert email is not None
183 assert year is not None
184 yield Contribution(
185 author=author, email=email, year=year, path=path)
186 author = email = year = None
187 blame.wait()
188 assert blame.returncode == 0
189
190
191 def get_source_paths(include: t.Set[str], exclude: t.Set[str])\
192 -> t.Iterator[Path]:
193 ls_tree = Popen(
194 ['git', 'ls-tree', '-r', '--name-only', 'HEAD'],
195 stdout=PIPE, stderr=DEVNULL, universal_newlines=True)
196 if not include:
197 include = {'*'}
198 if ls_tree.stdout is not None:
199 for filename in ls_tree.stdout:
200 filename = filename.strip()
201 if any(fnmatch(filename, pattern) for pattern in exclude):
202 continue
203 if any(fnmatch(filename, pattern) for pattern in include):
204 yield Path(filename)
205 ls_tree.wait()
206 assert ls_tree.returncode == 0
207
208
209 class License(t.NamedTuple):
210 ident: t.Optional[str]
211 text: t.List[str]
212
213
214 def get_license(path: Path, *, spdx_prefix: str = SPDX_PREFIX) -> License:
215 with open(path, 'r') as f:
216 lines = f.read().splitlines()
217
218 idents = [
219 line.rstrip() for line in lines
220 if line.startswith(spdx_prefix)
221 ]
222 ident = None
223 if len(idents) > 1:
224 raise RuntimeError(f'More than one {spdx_prefix} line in {path}!')
225 elif len(idents) == 1:
226 ident = idents[0]
227
228 body = [
229 line.rstrip() for line in lines
230 if not line.startswith(spdx_prefix)
231 ]
232 while not body[0]:
233 del body[0]
234 while not body[-1]:
235 del body[-1]
236 return License(ident, body)
237
238
239 class CopyWriter:
240 """
241 Transformer for the copyright header in source files. The :meth:`transform`
242 method can be called with a file-like object as the *source* and will
243 yield chunks of replacement data to be written to the replacement.
244 """
245
246 # The script's kinda dumb at this point - only handles straight-forward
247 # line-based comments, not multi-line delimited styles like /*..*/
248 COMMENTS = {
249 '': '#',
250 '.c': '//',
251 '.cpp': '//',
252 '.js': '//',
253 '.py': '#',
254 '.rst': '..',
255 '.sh': '#',
256 '.sql': '--',
257 }
258
259 def __init__(self, license: Path=Path('LICENSE.txt'),
260 preamble: t.List[str]=None,
261 spdx_prefix: str=SPDX_PREFIX,
262 copy_prefix: str=COPYRIGHT_PREFIX):
263 if preamble is None:
264 preamble = []
265 self.license = get_license(license, spdx_prefix=spdx_prefix)
266 self.preamble = preamble
267 self.spdx_prefix = spdx_prefix
268 self.copy_prefix = copy_prefix
269
270 @classmethod
271 def from_config(cls, config: Namespace) -> CopyWriter:
272 return cls(
273 config.license, config.preamble,
274 config.spdx_prefix, config.copy_prefix)
275
276 def transform(self, source: t.TextIO,
277 copyrights: t.List[Copyright], *,
278 comment_prefix: str=None) -> t.Iterator[str]:
279 if comment_prefix is None:
280 comment_prefix = self.COMMENTS[Path(source.name).suffix]
281 license_start = self.license.text[0]
282 license_end = self.license.text[-1]
283 state = 'header'
284 empty = True
285 for linenum, line in enumerate(source, start=1):
286 if state == 'header':
287 if linenum == 1 and line.startswith('#!'):
288 yield line
289 empty = False
290 elif linenum < 3 and (
291 'fileencoding=' in line or '-*- coding:' in line):
292 yield line
293 empty = False
294 elif line.rstrip() == comment_prefix:
295 pass # skip blank comment lines
296 elif line.startswith(f'{comment_prefix} {self.spdx_prefix}'):
297 pass # skip existing SPDX ident
298 elif line.startswith(f'{comment_prefix} {self.copy_prefix}'):
299 pass # skip existing copyright lines
300 elif any(line.startswith(f'{comment_prefix} {pre_line}')
301 for pre_line in self.preamble):
302 pass # skip existing preamble
303 elif line.startswith(f'{comment_prefix} {license_start}'):
304 state = 'license' # skip existing license lines
305 else:
306 yield from self._generate_header(
307 copyrights, comment_prefix, empty)
308 state = 'blank'
309 elif state == 'license':
310 if line.startswith(f'{comment_prefix} {license_end}'):
311 yield from self._generate_header(
312 copyrights, comment_prefix, empty)
313 state = 'blank'
314 continue
315 if state == 'blank':
316 # Ensure there's a blank line between license and start of the
317 # source body
318 if line.strip():
319 yield '\n'
320 yield line
321 state = 'body'
322 elif state == 'body':
323 yield line
324
325 def _generate_header(self, copyrights: t.Iterable[Copyright],
326 comment_prefix: str, empty: bool) -> t.Iterator[str]:
327 if not empty:
328 yield comment_prefix + '\n'
329 for line in self.preamble:
330 yield f'{comment_prefix} {line}\n'
331 if self.preamble:
332 yield comment_prefix + '\n'
333 for copyright in copyrights:
334 yield f'{comment_prefix} {self.copy_prefix} {copyright!s}\n'
335 yield comment_prefix + '\n'
336 if self.license.ident:
337 yield f'{comment_prefix} {self.license.ident}\n'
338 else:
339 for line in self.license.text:
340 if line:
341 yield f'{comment_prefix} {line}\n'
342 else:
343 yield comment_prefix + '\n'
344
345
346 class AtomicReplaceFile:
347 """
348 A context manager for atomically replacing a target file.
349
350 Uses :class:`tempfile.NamedTemporaryFile` to construct a temporary file in
351 the same directory as the target file. The associated file-like object is
352 returned as the context manager's variable; you should write the content
353 you wish to this object.
354
355 When the context manager exits, if no exception has occurred, the temporary
356 file will be renamed over the target file atomically (after copying
357 permissions from the target file). If an exception occurs during the
358 context manager's block, the temporary file will be deleted leaving the
359 original target file unaffected and the exception will be re-raised.
360
361 :param pathlib.Path path:
362 The full path and filename of the target file. This is expected to be
363 an absolute path.
364
365 :param str encoding:
366 If ``None`` (the default), the temporary file will be opened in binary
367 mode. Otherwise, this specifies the encoding to use with text mode.
368 """
369 def __init__(self, path: t.Union[str, Path], encoding: str = None):
370 if isinstance(path, str):
371 path = Path(path)
372 self._path = path
373 self._tempfile = tempfile.NamedTemporaryFile(
374 mode='wb' if encoding is None else 'w',
375 dir=str(self._path.parent), encoding=encoding, delete=False)
376 self._withfile = None
377
378 def __enter__(self):
379 self._withfile = self._tempfile.__enter__()
380 return self._withfile
381
382 def __exit__(self, exc_type, exc_value, exc_tb):
383 os.fchmod(self._withfile.file.fileno(), self._path.stat().st_mode)
384 result = self._tempfile.__exit__(exc_type, exc_value, exc_tb)
385 if exc_type is None:
386 os.rename(self._withfile.name, str(self._path))
387 else:
388 os.unlink(self._withfile.name)
389 return result
390
391
392 if __name__ == '__main__':
393 main()