Debian Codegraph @

upstream/mageia-cauldron/archive/update-packages.py

raw
1 #!/usr/bin/env python3
2 #
3 # Copyright © 2020 Dr. Tobias Quathamer <[email protected]>
4 #
5 # This program is free software: you can redistribute it and/or modify
6 # it under the terms of the GNU General Public License as published by
7 # the Free Software Foundation, either version 3 of the License, or
8 # (at your option) any later version.
9 #
10 # This program is distributed in the hope that it will be useful,
11 # but WITHOUT ANY WARRANTY; without even the implied warranty of
12 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13 # GNU General Public License for more details.
14 #
15 # You should have received a copy of the GNU General Public License
16 # along with this program. If not, see <http://www.gnu.org/licenses/>.
17
18 import json
19 import os
20 import re
21 import shutil
22 import subprocess
23 import sys
24 import urllib.request
25
26
27 # Read package information from a JSON file, if available
28 package_data = {}
29 if os.path.exists("downloads/packages.json"):
30 with open("downloads/packages.json") as input_file:
31 package_data = json.loads(input_file.read())
32
33 # Get current package information
34 download_page = urllib.request.urlopen(
35 "https://ftp-stud.hs-esslingen.de/pub/Mirrors/Mageia/distrib/cauldron/x86_64/media/core/release/"
36 )
37 contents = download_page.read().decode()
38 download_page.close()
39
40 # Read all packages names
41 with open("packages.txt") as input_file:
42 for line in input_file:
43 package = line.strip()
44
45 # Download HTML page and discover the correct link
46 print("Checking package '{}' ... ".format(package), end="")
47
48 package_url = re.findall(r"\"({}-[0-9].*?\.rpm)\"".format(package), contents)
49 if len(package_url) != 1:
50 print()
51 print(
52 "Error: Could not find URL for '{}'.".format(package), file=sys.stderr
53 )
54 print(
55 "If this is a permanent error, please remove the package from the file packages.txt.",
56 file=sys.stderr,
57 )
58 continue
59 else:
60 package_url = package_url[0]
61
62 package_url = "https://ftp-stud.hs-esslingen.de/pub/Mirrors/Mageia/distrib/cauldron/x86_64/media/core/release/{}".format(
63 package_url
64 )
65 package_filename = re.split(r"/", package_url)[-1]
66
67 # If the latest download is already available, skip this package
68 if package in package_data:
69 if package_data[package]["filename"] == package_filename:
70 print("current.")
71 continue
72
73 # Store information about downloaded package
74 package_data[package] = {
75 "filename": package_filename,
76 "needs_templates": True,
77 "needs_update": True,
78 }
79
80 # Prepare for downloading the file
81 package_dir = "downloads/{}".format(package)
82 local_filename = "downloads/{}/{}".format(package, package_filename)
83
84 # Ensure there are no leftovers of previous packages
85 os.makedirs(package_dir, exist_ok=True)
86 shutil.rmtree(package_dir)
87 os.makedirs(package_dir, exist_ok=True)
88
89 # Download new package
90 urllib.request.urlretrieve(package_url, local_filename)
91 print("downloaded.")
92
93 # Get the exact filename of data.tar with the extension
94 os.makedirs("{}/tmp".format(package_dir), exist_ok=True)
95
96 # Extract the package into a temporary file
97 with open("{}/tmp/tmpcpio".format(package_dir), "w") as outfile:
98 subprocess.run(["rpm2cpio", local_filename], stdout=outfile)
99 with open("{}/tmp/tmpcpio".format(package_dir), "r") as infile:
100 subprocess.run(
101 ["bsdcpio", "--quiet", "-i", "--make-directories"],
102 cwd="{}/tmp".format(package_dir),
103 stdin=infile,
104 )
105
106 # Move the manpages up
107 manpath = "{}/tmp/usr/share/man".format(package_dir)
108 for mandir in os.listdir(manpath):
109 if not mandir.startswith("man"):
110 continue
111 # Make sure the directory exists
112 os.makedirs(mandir, exist_ok=True)
113 shutil.move("{}/{}".format(manpath, mandir), "{}".format(package_dir))
114
115 # Cleanups
116 shutil.rmtree("downloads/{}/tmp".format(package))
117 os.remove(local_filename)
118
119 # Save package information in a JSON file
120 with open("downloads/packages.json", "w") as output_file:
121 output_file.write(json.dumps(package_data, sort_keys=True, indent=4))