Debian Codegraph @

upstream/debian-unstable/archive/update-packages.py

raw
1 #!/usr/bin/env python3
2 #
3 # Copyright © 2020 Dr. Tobias Quathamer <[email protected]>
4 #
5 # This program is free software: you can redistribute it and/or modify
6 # it under the terms of the GNU General Public License as published by
7 # the Free Software Foundation, either version 3 of the License, or
8 # (at your option) any later version.
9 #
10 # This program is distributed in the hope that it will be useful,
11 # but WITHOUT ANY WARRANTY; without even the implied warranty of
12 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13 # GNU General Public License for more details.
14 #
15 # You should have received a copy of the GNU General Public License
16 # along with this program. If not, see <http://www.gnu.org/licenses/>.
17
18 import json
19 import os
20 import re
21 import shutil
22 import subprocess
23 import sys
24 import urllib.request
25
26
27 # Read package information from a JSON file, if available
28 package_data = {}
29 if os.path.exists("downloads/packages.json"):
30 with open("downloads/packages.json") as input_file:
31 package_data = json.loads(input_file.read())
32
33 # Read all packages names
34 with open("packages.txt") as input_file:
35 for package in input_file:
36 package = package.strip()
37 # Download HTML page and discover the correct link
38 print("Checking package '{}' ... ".format(package), end="")
39
40 download_page = urllib.request.urlopen(
41 "https://packages.debian.org/sid/amd64/{}/download".format(package)
42 )
43 contents = download_page.read().decode()
44 download_page.close()
45
46 package_url = re.findall(
47 r"https?://ftp\.de\.debian\.org/debian/pool/[^\"]*", contents
48 )
49 if len(package_url) != 1:
50 print()
51 print(
52 "Error: Could not find URL for '{}'.".format(package), file=sys.stderr
53 )
54 print(
55 "If this is a permanent error, please remove the package from the file packages.txt.",
56 file=sys.stderr,
57 )
58 continue
59 else:
60 package_url = package_url[0]
61
62 package_filename = re.split(r"/", package_url)[-1]
63
64 # If the latest download is already available, skip this package
65 if package in package_data:
66 if package_data[package]["filename"] == package_filename:
67 print("current.")
68 continue
69
70 # Store information about downloaded package
71 package_data[package] = {
72 "filename": package_filename,
73 "needs_templates": True,
74 "needs_update": True,
75 }
76
77 # Prepare for downloading the file
78 package_dir = "downloads/{}".format(package)
79 local_filename = "downloads/{}/{}".format(package, package_filename)
80
81 # Ensure there are no leftovers of previous packages
82 os.makedirs(package_dir, exist_ok=True)
83 shutil.rmtree(package_dir)
84 os.makedirs(package_dir, exist_ok=True)
85
86 # Download new package
87 urllib.request.urlretrieve(package_url, local_filename)
88 print("downloaded.")
89
90 # Get the exact filename of data.tar with the extension
91 os.makedirs("{}/tmp".format(package_dir), exist_ok=True)
92 output = subprocess.run(["ar", "t", local_filename], capture_output=True)
93 data_tar = re.findall(r"data\.tar.*", output.stdout.decode())[0]
94
95 # Extract the package data
96 subprocess.run(["ar", "x", local_filename, data_tar])
97
98 # Extract the package into a temporary directory
99 subprocess.run(
100 ["tar", "xaf", data_tar, "--directory={}/tmp".format(package_dir)]
101 )
102
103 # Move the manpages up
104 manpath = "{}/tmp/usr/share/man".format(package_dir)
105 for mandir in os.listdir(manpath):
106 if not mandir.startswith("man"):
107 continue
108 # Make sure the directory exists
109 os.makedirs(mandir, exist_ok=True)
110 shutil.move("{}/{}".format(manpath, mandir), "{}".format(package_dir))
111
112 # Cleanups
113 os.remove(data_tar)
114 shutil.rmtree("downloads/{}/tmp".format(package))
115 os.remove(local_filename)
116
117 # Save package information in a JSON file
118 with open("downloads/packages.json", "w") as output_file:
119 output_file.write(json.dumps(package_data, sort_keys=True, indent=4))