Debian Codegraph @

upstream/fedora-44/archive/update-packages.py

raw
1 #!/usr/bin/env python3
2 #
3 # Copyright © 2020 Dr. Tobias Quathamer <[email protected]>
4 #
5 # This program is free software: you can redistribute it and/or modify
6 # it under the terms of the GNU General Public License as published by
7 # the Free Software Foundation, either version 3 of the License, or
8 # (at your option) any later version.
9 #
10 # This program is distributed in the hope that it will be useful,
11 # but WITHOUT ANY WARRANTY; without even the implied warranty of
12 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13 # GNU General Public License for more details.
14 #
15 # You should have received a copy of the GNU General Public License
16 # along with this program. If not, see <http://www.gnu.org/licenses/>.
17
18 import json
19 import os
20 import re
21 import shutil
22 import subprocess
23 import sys
24 import urllib.request
25
26
27 # Read package information from a JSON file, if available
28 package_data = {}
29 if os.path.exists("downloads/packages.json"):
30 with open("downloads/packages.json") as input_file:
31 package_data = json.loads(input_file.read())
32
33 # Read all packages names
34 page_buffer = {}
35 with open("packages.txt") as input_file:
36 for line in input_file:
37 package = line.strip()
38 first_letter = package[0].lower()
39
40 # Download HTML page and discover the correct link
41 print("Checking package '{}' ... ".format(package), end="")
42
43 if not first_letter in page_buffer:
44 download_page = urllib.request.urlopen(
45 "https://ftp.fau.de/fedora/linux/development/rawhide/Everything/x86_64/os/Packages/{}".format(
46 first_letter
47 )
48 )
49 page_buffer[first_letter] = download_page.read().decode()
50 download_page.close()
51 contents = page_buffer[first_letter]
52
53 package_url = re.findall(r"\"({}-[0-9].*?\.rpm)\"".format(package), contents)
54 if len(package_url) != 1:
55 # There are multiple matches, usually this should be
56 # i686 and x86_64. We care only for the second one.
57 found_64 = False
58 for item in package_url:
59 if "x86_64" in item:
60 package_url = item
61 found_64 = True
62 break
63 if not found_64:
64 print()
65 print(
66 "Error: Could not find URL for '{}'.".format(package),
67 file=sys.stderr,
68 )
69 print(
70 "If this is a permanent error, please remove the package from the file packages.txt.",
71 file=sys.stderr,
72 )
73 continue
74 else:
75 package_url = package_url[0]
76
77 package_url = "https://ftp.fau.de/fedora/linux/development/rawhide/Everything/x86_64/os/Packages/{}/{}".format(
78 first_letter, package_url
79 )
80 package_filename = re.split(r"/", package_url)[-1]
81
82 # If the latest download is already available, skip this package
83 if package in package_data:
84 if package_data[package]["filename"] == package_filename:
85 print("current.")
86 continue
87
88 # Store information about downloaded package
89 package_data[package] = {
90 "filename": package_filename,
91 "needs_templates": True,
92 "needs_update": True,
93 }
94
95 # Prepare for downloading the file
96 package_dir = "downloads/{}".format(package)
97 local_filename = "downloads/{}/{}".format(package, package_filename)
98
99 # Ensure there are no leftovers of previous packages
100 os.makedirs(package_dir, exist_ok=True)
101 shutil.rmtree(package_dir)
102 os.makedirs(package_dir, exist_ok=True)
103
104 # Download new package
105 urllib.request.urlretrieve(package_url, local_filename)
106 print("downloaded.")
107
108 # Get the exact filename of data.tar with the extension
109 os.makedirs("{}/tmp".format(package_dir), exist_ok=True)
110
111 # Extract the package into a temporary file
112 with open("{}/tmp/tmpcpio".format(package_dir), "w") as outfile:
113 subprocess.run(["rpm2cpio", local_filename], stdout=outfile)
114 with open("{}/tmp/tmpcpio".format(package_dir), "r") as infile:
115 subprocess.run(
116 ["bsdcpio", "--quiet", "-i", "--make-directories"],
117 cwd="{}/tmp".format(package_dir),
118 stdin=infile,
119 )
120
121 # Move the manpages up
122 manpath = "{}/tmp/usr/share/man".format(package_dir)
123 for mandir in os.listdir(manpath):
124 if not mandir.startswith("man"):
125 continue
126 # Make sure the directory exists
127 os.makedirs(mandir, exist_ok=True)
128 shutil.move("{}/{}".format(manpath, mandir), "{}".format(package_dir))
129
130 # Cleanups
131 shutil.rmtree("downloads/{}/tmp".format(package))
132 os.remove(local_filename)
133
134 # Save package information in a JSON file
135 with open("downloads/packages.json", "w") as output_file:
136 output_file.write(json.dumps(package_data, sort_keys=True, indent=4))