Debian Codegraph @

debian/patches/add_debian_management_script

raw
1 Description: Add debian maintainer assistant
2 This patch add a helper script for maintainers of Debian FreeDict packages. See
3 the documentation of that script to find out more.
4 Author: Sebastian Humenda <[email protected]>
5 Last-Update: 2018-10-28
6
7 Index: freedict-tools/fetchdictdata.py
8 ===================================================================
9 --- /dev/null
10 +++ freedict-tools/fetchdictdata.py
11 @@ -0,0 +1,619 @@
12 +#!/usr/bin/env python3
13 +"""This script is designed to help with the Debian FreeDict packaging. It can
14 +fetch all available databases from
15 +https://www.freedict.org/freedict-database.xml
16 +and generate a orig source tar ball or generate debian/control and
17 +debian/copyright.
18 +
19 +Please use `--help` to find out more details.
20 +
21 +IMPORTANT: you must run this script from the root of the freedict source,
22 +otherwise the operations will fail.
23 +
24 +Generating debian/copyright and debian/control
25 +==============================================
26 +
27 +Since FreeDict packages contain a lot of very similar dictionaries, the process
28 +is made more convenient by generating the mentioned files. d/control is made up
29 +of a d/control.HEAD file (make sure this one exists), usually with the normal
30 +source stanza and followed by auto-generated stanzas, derived from information
31 +of the debian/freedict-database.xml. The XML API file is fetched automatically,
32 +if required.
33 +
34 +The d/copyright file is generated using licensecheck and a bit of guessing
35 +logic. The file d/copyright.snippets/TAIL is appended and usually contains the
36 +licence definition stanzas. If the copyright file contains "FIXME", the
37 +maintainer may also like to create a file d/copyright.snippets/xxx-yyy plain
38 +text As soon as such a file is found, no licence information will be queried for
39 +the specified dictionary.
40 +"""
41 +
42 +# pylint: disable=multiple-imports,too-few-public-methods,global-statement
43 +import argparse
44 +import datetime
45 +import os
46 +import re
47 +import shutil
48 +import subprocess
49 +import sys
50 +import textwrap
51 +import urllib.request
52 +import xml.etree.ElementTree as ET
53 +from packaging.version import Version
54 +
55 +XML_URL = "http://www.freedict.org/freedict-database.xml"
56 +XML_API_PATH = "debian/freedict-database.xml"
57 +__DICT2LONGNAME = {} # filled by dictionarycode2longdescription
58 +
59 +
60 +def dictionarycode2longdescription(string):
61 + """Translate a dictionary code into a human-readable string.
62 + A dictionary is encoded using a hyphen-delimited pair of ISO 639-3 codes.
63 + This function translates this into a language pair of English names,
64 + delimited by a hyphen."""
65 + global __DICT2LONGNAME
66 + if not __DICT2LONGNAME:
67 + __DICT2LONGNAME = parse_iso_table()
68 + string = string.split("-")
69 + return __DICT2LONGNAME[string[0]] + "-" + __DICT2LONGNAME[string[1]]
70 +
71 +
72 +def parse_iso_table():
73 + """Parse the ISO codes from a tab-delimited data file in debian/ named
74 + iso*.tab."""
75 + tbl = {}
76 + for item in os.listdir("debian"):
77 + if item.startswith("iso-"):
78 + with open(os.path.join("debian", item), "r", encoding="utf-8") as tbl_file:
79 + tmp = tbl_file.read()
80 + tmp = tmp.split("\n")
81 + for line in tmp[1:]:
82 + line = line.split("\t")
83 + name = line[6]
84 + # remove parenthesized vom language name
85 + name = re.sub(r"\s*\(.*?\)\s*", "", name)
86 + tbl[line[0]] = name
87 + if line[1] != "": # some have two language code, add it:
88 + tbl[line[1]] = name
89 + return tbl
90 +
91 +
92 +def get_xml_content(fetch_new=False):
93 + """Either read contents of freedict-database.xml or fetch it from the web
94 +first and save it."""
95 + if fetch_new or not os.path.exists(XML_API_PATH):
96 + print(f"Downloading API from {XML_URL}")
97 + with urllib.request.urlopen(XML_URL) as req:
98 + data = req.read().decode("utf-8")
99 + with open(XML_API_PATH, "w", encoding="UTF-8") as file:
100 + file.write(data)
101 + else: # read from file
102 + with open(XML_API_PATH, encoding="UTF-8") as file:
103 + data = file.read()
104 + return data
105 +
106 +def run_licensecheck_on(file_path):
107 + """Try to extract licence with licence check."""
108 + with subprocess.Popen(
109 + ["licensecheck", "--copyright", "--deb-fmt", file_path], stdout=subprocess.PIPE
110 + ) as proc:
111 + try:
112 + text = proc.communicate()[0].decode("utf-8")
113 + text = text.lstrip().split("\n", maxsplit=1)[0]
114 + except IndexError:
115 + return None
116 + sre = re.search(r"^.+: (.+)$", text)
117 + if not sre:
118 + sre = re.search(r"^.No copyright. (.+)$", text.split("\n")[0])
119 + if not sre:
120 + return None
121 + match = sre.groups()[0]
122 + return (None if "UNKNOWN" in match else match)
123 +
124 +def find_licence(dictionary):
125 + """Find out licence of dictionary.
126 +First run licence-check, afterwards use a self-brewed licence checker.
127 +Currently only GPL is detected, else FIXMe is output. It tries to output
128 +something like GPL, GPL-2, GPL-3, GPL-2+, GPL-3+"""
129 +
130 +
131 + tei_fn = "{0}{1}{0}.{2}".format(dictionary.name, os.sep, "tei")
132 + licence = run_licensecheck_on(tei_fn)
133 + if not licence:
134 + licence = run_licensecheck_on(dictionary.name + os.sep + "COPYING.tei")
135 +
136 + if not licence:
137 + licence = "FIXME" # backup solution
138 +
139 + if licence != "FIXME": # found a licence, return
140 + NO_COPYRIGHT_BY_LICENSECHECK = "*No copyright* "
141 + if licence.startswith(NO_COPYRIGHT_BY_LICENSECHECK):
142 + licence = licence[len(NO_COPYRIGHT_BY_LICENSECHECK):]
143 + return licence
144 +
145 + # try guessing the licence from the TEI file
146 + with open(tei_fn, "r", encoding="utf-8") as f:
147 + in_header = True
148 + line = "start"
149 + lastline = ""
150 + while in_header and line != "":
151 + line = f.readline().lower()
152 + if "<body" in line:
153 + line = line[: line.find("<body")] # parse everything before
154 + in_header = False
155 + if "gpl" in line.lower() or "gnu general public lic" in line.lower():
156 + licence = "GPL"
157 + # try to extract version number
158 + res = re.search(
159 + r"(?:version|ver.|licence|licence)\s+(\d+)", lastline + line.lower()
160 + )
161 + if res:
162 + licence += "-%s" % res.groups()[0]
163 + elif "attribution-sharealike" in line:
164 + licence = "CC-BY-SA"
165 + version = re.search(r"sharealike \(?v?(\d+\.?\d*)", line)
166 + if version:
167 + licence += "-%s" % version.groups()[0]
168 + lastlines = lastline + line
169 + if licence.startswith("GPL") and not licence.endswith("+"):
170 + if (
171 + re.search(".*later.*version", lastlines)
172 + or "or later" in lastlines
173 + or "and any later" in lastlines
174 + ):
175 + licence += "+"
176 + lastline = line[:]
177 + return licence
178 +
179 +
180 +def recursive_text(node):
181 + text = ""
182 + if not node.text.strip() == "":
183 + text = node.text
184 + for child in node:
185 + text += "\n" + recursive_text(child)
186 + return text
187 +
188 +class Dictionary:
189 + """A `named tuple` to represent a dictionary from the API."""
190 + def __init__(self, name):
191 + self.name = name
192 + self.headwords, self.version = None, None
193 + self.status, self.maintainer, self.source_release = None, None, None
194 +
195 + def set(self, key, val):
196 + """Map the XML attribute names to the internal name."""
197 + match key:
198 + case "headwords": self.headwords = val
199 + case "edition": self.version = val
200 + case "status": self.status = val
201 + case "maintainerName": self.maintainer = val
202 + case _: raise ValueError(f"invalid key: {key}")
203 +
204 +class Action:
205 + """An action to perform on the FD API."""
206 + def __init__(self, package_name, dictionaries):
207 + self.dictionaries = dictionaries
208 + # name of the debian (source) package
209 + self.package_name = package_name
210 +
211 + def perform_action(self):
212 + pass
213 +
214 +class GenerateControlCopyright(Action):
215 + def __init__(self, package_name, dictionaries, no_desc_version):
216 + super().__init__(package_name, dictionaries)
217 + # keep/strip version number from pkg description
218 + self.__desc_version = not no_desc_version
219 +
220 + def perform_action(self):
221 + """Write both control as well as the copyright file."""
222 + self.write_control()
223 + self.write_copyright()
224 +
225 + def gen_control_for_dictionary(self, platform, dictionary, depends=None,
226 + recommends = None, suggests=None, provides=None):
227 + """Generate a list of tokens that, if joined by "" resembles a packet description."""
228 + tokens = [f"\nPackage: {platform}-freedict-{dictionary.name}\n"]
229 + tokens.append(f"Architecture: all\n")
230 + tokens.append("Depends: ${misc:Depends}")
231 + tokens.append(f" {depends}" if depends else "\n")
232 + if recommends:
233 + tokens.append(f"Recommends: {recommends}\n")
234 + if suggests:
235 + tokens.append(f"Suggests: {suggests}\n")
236 + if provides:
237 + tokens.append(f"Provides: {provides}\n")
238 + status = ""
239 + if dictionary.status:
240 + status = f" (FreeDict status: {dictionary.status})"
241 + longname = dictionarycode2longdescription(dictionary.name)
242 + tokens.append(
243 + f"Description: {longname} dictionary\n"
244 + )
245 + version = ""
246 + if self.__desc_version:
247 + version = f", version {dictionary.version}"
248 + longdesc = (
249 + f"This is the {longname} dictionary from the FreeDict "
250 + f"project{version}. It contains {dictionary.headwords} "
251 + f"headwords{status}. It can be used with various dictionary "
252 + "applications."
253 + )
254 + # format description to 80 characters per line
255 + tokens.append(
256 + "\n".join(
257 + textwrap.wrap(
258 + longdesc, width=79, initial_indent=" ", subsequent_indent=" "
259 + )
260 + )
261 + )
262 + tokens.append("\n")
263 + return tokens
264 +
265 + def write_control(self):
266 + """Generate debian/control from debian/control.HEAD and the gathered
267 + dictionary data."""
268 + HEAD = open("debian/control.HEAD", "r", encoding="utf-8").read() + "\n"
269 + tokens = [HEAD.rstrip(), "\n\n"]
270 +
271 + for dictionary in self.dictionaries:
272 + tokens.extend(self.gen_control_for_dictionary("dict", dictionary,
273 + suggests = ("dictd | dicod, dict | goldendict "
274 + "| dictionaryreader.app"),
275 + provides = "dictd-dictionary"))
276 + tokens.extend(self.gen_control_for_dictionary("stardicct",
277 + dictionary,
278 + recommends = ("sdcv (>= 0.5~) | stardict (>= 3.0~) "
279 + "| stardict-gtk (>= 3.0~) | goldendict | qstardict")))
280 +
281 + with open("debian/control", "w", encoding="UTF-8") as file:
282 + file.write("".join(tokens))
283 +
284 + def write_copyright(self):
285 + """Generate debian/copyright from debian/copyright.HEAD and the gathered
286 + dictionary data."""
287 + upstream_last_touched = int(
288 + subprocess.check_output(["dpkg-parsechangelog", "-S", "version"])
289 + .decode(sys.getdefaultencoding())
290 + .split(".", 1)[0]
291 + .strip()
292 + )
293 + cprght_snippets = "{1}{0}{2}{0}".format(os.sep, "debian", "copyright.snippets")
294 + HEAD = open(cprght_snippets + "HEAD", encoding="utf-8").read()
295 + string = [HEAD, "\n"]
296 + for dictionary in self.dictionaries:
297 + # is there a snippet for an exception to licensecheck?
298 + exception_snippet = cprght_snippets + dictionary.name
299 + if os.path.exists(exception_snippet):
300 + with open(exception_snippet) as f:
301 + string.append("\n" + f.read())
302 + else:
303 + string.append(f"\nFiles: {dictionary.name}/*\n")
304 + string.append(
305 + f"Copyright: 2000-{upstream_last_touched} FreeDict contributors\n"
306 + )
307 + string.append(f"License: {find_licence(dictionary)}\n")
308 + with open(cprght_snippets + "TAIL", encoding="UTF-8") as f:
309 + string += ["\n\n", f.read()]
310 +
311 + document = "".join(string)
312 + with open("debian/copyright", "w", encoding="utf-8") as f:
313 + f.write(document)
314 + if "FIXME" in document:
315 + print(
316 + 'NOTE: some licences could not be extracted, search for "FIXME" in debian/copyright.'
317 + )
318 +
319 +
320 +class FetchSource(Action):
321 + """Fetch the sources of all dictionaries and the tools directory."""
322 +
323 + def __init__(self, package_name, dictionaries):
324 + super().__init__(package_name, dictionaries)
325 + self.date = self.gen_date()
326 + self.dirname = f"{package_name}-{self.date}.orig"
327 + self.exclude_dictionaries = []
328 + if len(sys.argv) == 4: # there's the -x option given
329 + if sys.argv[2] == "-x":
330 + self.exclude_dictionaries = sys.argv[3].split(" ")
331 +
332 + def gen_date(self):
333 + """Return date in format "yyyy.mm.dd"."""
334 + d = datetime.datetime.now()
335 + return (
336 + str(d.year)
337 + + "."
338 + + str(d.month).zfill(2).replace(" ", "0")
339 + + "."
340 + + str(d.day).zfill(2).replace(" ", "0")
341 + )
342 +
343 + def prepare_environment(self):
344 + """Perform all actions which are needed before downloading the
345 + source.
346 + """
347 + if os.path.exists(self.dirname):
348 + print(
349 + "Removing %s; possibly left over from an interrupted run."
350 + % self.dirname
351 + )
352 + shutil.rmtree(self.dirname)
353 + os.mkdir(self.dirname)
354 + os.chdir(self.dirname)
355 +
356 + def clean_up(self):
357 + """
358 + Compress the original source, move it to the right destination and
359 + remove download directory."""
360 + tarname = f"{self.package_name}_{self.date}.orig.tar.xz"
361 + os.chdir("..")
362 + ret = subprocess.call(["tar", "cJf", tarname, self.dirname])
363 + if ret:
364 + sys.exit(9)
365 + print("Moving tar archive upward to.", os.path.join("..", tarname))
366 + os.rename(tarname, ".." + os.sep + tarname)
367 + shutil.rmtree(self.dirname)
368 +
369 + def perform_action(self):
370 + """Download all upstream source packages."""
371 + self.prepare_environment()
372 + imported = 0
373 + tarname = f"{self.package_name}_{self.date}.orig.tar.xz"
374 + for dictionary in self.dictionaries:
375 + if dictionary.name in self.exclude_dictionaries:
376 + print(f"Skip {dictionary.name} (specified via commmand line)")
377 + continue
378 + elif dictionary.source_release is None:
379 + print(f"Skip {dictionary.name} (no source release)")
380 + continue
381 + fname = dictionary.source_release.split("/")[-1]
382 + print(f"Fetching {dictionary.name} from {dictionary.source_release}")
383 + try:
384 + with urllib.request.urlopen(dictionary.source_release) as u:
385 + data = u.read()
386 + except urllib.error.HTTPError as h:
387 + if int(h.code) == 404:
388 + reason = f"{str(h)}: {dictionary.source_release}"
389 + raise urllib.error.URLError(reason) from None
390 + raise h from None
391 +
392 + with open(fname, "wb") as f:
393 + f.write(data)
394 + print("Extracting", fname)
395 + if fname.endswith(".zip"):
396 + os.system('unzip -qq "%s"' % fname)
397 + elif any(
398 + fname.endswith(suf) for suf in (".tar.bz2", ".tar.gz", ".tar.xz")
399 + ):
400 + subprocess.run(["tar", "xf", fname], check=True)
401 + else:
402 + print(f'E: unknown format of "{fname}".')
403 + sys.exit(0)
404 +
405 + subprocess.run(["tar", "xf", fname], check=True)
406 + os.remove(fname)
407 + imported += 1
408 + print("Imported %d dictionaries." % imported)
409 + self.clean_up()
410 +
411 +class CheckForMoreRecentDicts(Action):
412 + """Check whether the downstream copies are outdated."""
413 + def __init__(self, package_name, dictionaries):
414 + super().__init__(package_name, dictionaries)
415 +
416 + def perform_action(self):
417 + """Check whether any dictionary is more recent upstream."""
418 + version_regex = re.compile("<edition>(.+?)</edition>")
419 + def version_of(v):
420 + return Version(v.split("-")[0])
421 + updates_required = []
422 + for dictionary in self.dictionaries:
423 + version = None
424 + with open(f"{dictionary.name}/{dictionary.name}.tei",
425 + encoding="UTF-8") as file:
426 + for line in file:
427 + match = version_regex.search(line)
428 + if match:
429 + version = match.groups()[0]
430 + break
431 + if not version:
432 + raise ValueError(f"unable to get version from {dictionary.name}")
433 + if version_of(version) < version_of(dictionary.version):
434 + updates_required.append((dictionary, version))
435 + if updates_required:
436 + print("Updates required for:")
437 + for dictionary, old_version in updates_required:
438 + print(f" {dictionary.name}: old {old_version}, new {dictionary.version}")
439 + else:
440 + print("All dictionaries up-to-date.")
441 +
442 +class Criteria:
443 + """A criteria matcher: Use the string
444 + sourceURL:wikdict
445 + to filter for all dictionaries fwith wikdict in their URL and use
446 + sourceURL!wikdict
447 + to filter for all dictionaries not derived from wikdict. The first part is
448 + an XML attribute name, the second bit is a regular expression.
449 + This class can cope with empty criteria: all matches will be true."""
450 +
451 + def __init__(self, criteria):
452 + self.__rgx = None
453 + self.__delim = None
454 + self.__criteria_attr = None
455 + if criteria and (not ":" in criteria and not "!" in criteria):
456 + raise ValueError(
457 + "criteria needs to consist of a dictionary "
458 + "attribute followed by `:` or `!` followed by a regular "
459 + "expression"
460 + )
461 + if criteria:
462 + self.__delim = ":" if ":" in criteria else "!"
463 + self.__criteria_attr, self.__rgx = criteria.split(self.__delim, 1)
464 + self.__rgx = re.compile(self.__rgx)
465 +
466 + def matches(self, dictnode):
467 + """Test whether given dictnode matches configured criteria."""
468 + if not self.__criteria_attr:
469 + return True # no criteria to match on, include unconditionally
470 + attr = dictnode.get(self.__criteria_attr)
471 + if not attr:
472 + # **only** include positive matches, yet nothing to match on found
473 + return not self.__delim == ":"
474 + if self.__delim == ":":
475 + return bool(self.__rgx.search(attr))
476 + return not bool(self.__rgx.search(attr))
477 +
478 +
479 +def clean_up_tree(root, criteria):
480 + """Iterate over XML tree and delete those <dictionary/>-nodes which have no
481 + release or which don't match one of the given criteria."""
482 + criteria_matched = False # track whether a criteria was matchd
483 + criteria = Criteria(criteria)
484 + dictionary_idx = 0
485 + dictionaries = list(root)
486 + while dictionary_idx < len(dictionaries) - 1:
487 + dictionary = dictionaries[dictionary_idx]
488 + # criteria matched and has children (releases)
489 + if criteria.matches(dictionary) and list(dictionary):
490 + criteria_matched = True
491 + dictionary_idx += 1
492 + else:
493 + print(f'Skipping {dictionary.attrib["name"]}, ', end="")
494 + print(("no releases" if not list(dictionary) else "didn't match criteria"))
495 + root.remove(dictionary)
496 + dictionaries = list(root)
497 + if not criteria_matched:
498 + print("Warning: given criteria never matched")
499 +
500 +
501 +def parse_api(root) -> list[str, dict]:
502 + """Parse all dictionaries from the given XML FreeDict API. Returns a sorted
503 + list (by ISO-639-3 dictionary name)."""
504 + dictionaries = []
505 + for child in root:
506 + if not list(child) or not child.attrib.get("name") \
507 + or not child.tag.endswith("dictionary"):
508 + continue # skip dictionaries without releases or non-dictionary nodes
509 + dictionary = Dictionary(child.attrib["name"])
510 + for key in ("headwords", "edition", "status", "maintainerName"):
511 + try:
512 + dictionary.set(key, child.attrib[key])
513 + except KeyError as e:
514 + if e.args[0] != "status":
515 + raise KeyError(
516 + "missing attribute for %s: %s"
517 + % (child.attrib["name"], e.args[0])
518 + ) from None
519 + for release in child:
520 + if not release.attrib.get("platform") \
521 + or release.attrib["platform"] != "src":
522 + continue
523 + dictionary.source_release = release.attrib["URL"]
524 + break # only one release
525 + dictionaries.append(dictionary)
526 + return dictionaries
527 +
528 +
529 +def parse_args():
530 + if os.getcwd().endswith("debian"):
531 + os.chdir("..")
532 + if not any(re.match(r"[a-z]{3}-[a-z]{3}", f) for f in os.listdir(os.getcwd())):
533 + print("You must run this script from the FreeDict packaging root.")
534 + sys.exit(127)
535 + parser = argparse.ArgumentParser(description=__doc__)
536 + parser.add_argument(
537 + "--criteria",
538 + dest="criteria",
539 + default=None,
540 + help=(
541 + "A criteria is a XML attribute name (from the Dictionary tag),"
542 + " separated by a delimiter, followed by a regular expression. "
543 + "This can be used to filter dictionaries. If attribute name and"
544 + " regular expression are separated by a colon `:`, the "
545 + "dictionary node MUST match the given expression, if it "
546 + "separated by a exclamation mark `!`, these dictionaries will "
547 + "be dropped."
548 + ),
549 + )
550 + parser.add_argument(
551 + "--dc",
552 + dest="gen_control_copyright",
553 + action="store_true",
554 + default=False,
555 + help="generate debian/copyright and debian/control",
556 + )
557 + parser.add_argument(
558 + "--need-update",
559 + dest="check_for_upstream_dict_updates",
560 + action="store_true",
561 + default=False,
562 + help="check whether any package needs an update (implies -u",
563 + )
564 + parser.add_argument(
565 + "--no-desc-version",
566 + dest="no_desc_version",
567 + action="store_true",
568 + default=False,
569 + help="omit the version number in package descriptions (only "
570 + "useful when using --dc)",
571 + )
572 + parser.add_argument(
573 + "--orig",
574 + dest="fetch_orig",
575 + action="store_true",
576 + default=False,
577 + help="fetch a new orig source tar ball to ../",
578 + )
579 + parser.add_argument(
580 + "-u",
581 + dest="update_xml_api",
582 + action="store_true",
583 + default=False,
584 + help="Update FreeDict XML API file and exit.",
585 + )
586 + if len(sys.argv) == 1:
587 + parser.print_usage()
588 + return parser.parse_args()
589 +
590 +
591 +def main():
592 + # are we in the correct directory?
593 + # cmd args
594 + args = parse_args()
595 + # perform an update of the XML API if requested *or* if the check for newer
596 + # upstream dictionaries is requested
597 + perform_xml_api_update = (True if args.check_for_upstream_dict_updates
598 + else args.update_xml_api)
599 + xmlsrc = get_xml_content(perform_xml_api_update)
600 + actions = []
601 + # must be checked first
602 + # actions can be combined
603 + if args.fetch_orig:
604 + xmlsrc = get_xml_content(fetch_new=True) # fetch latest FreeDict API file
605 + actions += [FetchSource]
606 + if args.gen_control_copyright:
607 + actions += [GenerateControlCopyright]
608 + if args.check_for_upstream_dict_updates:
609 + actions += [CheckForMoreRecentDicts]
610 + package_name = subprocess.check_output(["dpkg-parsechangelog",
611 + "-S", "Source"]) \
612 + .decode(sys.getdefaultencoding()) \
613 + .strip()
614 +
615 + # usual operation
616 + root = ET.fromstring(xmlsrc)
617 + clean_up_tree(root, criteria=args.criteria)
618 + dictionaries = parse_api(root)
619 + for obj in actions:
620 + inst = None
621 + if obj == GenerateControlCopyright:
622 + #pylint: disable=too-many-function-args
623 + inst = obj(package_name, dictionaries, args.no_desc_version)
624 + else:
625 + inst = obj(package_name, dictionaries)
626 + inst.perform_action()
627 +
628 +
629 +if __name__ == "__main__":
630 + main()