Debian Codegraph @

upstream/opensuse-tumbleweed/archive/update-packages.py

raw
1 #!/usr/bin/env python3
2 #
3 # Copyright © 2020 Dr. Tobias Quathamer <[email protected]>
4 #
5 # This program is free software: you can redistribute it and/or modify
6 # it under the terms of the GNU General Public License as published by
7 # the Free Software Foundation, either version 3 of the License, or
8 # (at your option) any later version.
9 #
10 # This program is distributed in the hope that it will be useful,
11 # but WITHOUT ANY WARRANTY; without even the implied warranty of
12 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13 # GNU General Public License for more details.
14 #
15 # You should have received a copy of the GNU General Public License
16 # along with this program. If not, see <http://www.gnu.org/licenses/>.
17
18 import json
19 import os
20 import re
21 import shutil
22 import subprocess
23 import sys
24 import urllib.request
25
26
27 # Read package information from a JSON file, if available
28 package_data = {}
29 if os.path.exists("downloads/packages.json"):
30 with open("downloads/packages.json") as input_file:
31 package_data = json.loads(input_file.read())
32
33 # Get current package information
34 download_page = urllib.request.urlopen(
35 "https://ftp.gwdg.de/pub/opensuse/tumbleweed/repo/oss/x86_64/"
36 )
37 contents = download_page.read().decode()
38 download_page.close()
39
40 # Also get noarch package information as a fallback
41 download_page = urllib.request.urlopen(
42 "https://ftp.gwdg.de/pub/opensuse/tumbleweed/repo/oss/noarch/"
43 )
44 contents_fallback = download_page.read().decode()
45 download_page.close()
46
47 # Read all packages names
48 with open("packages.txt") as input_file:
49 for line in input_file:
50 package = line.strip()
51
52 # Download HTML page and discover the correct link
53 print("Checking package '{}' ... ".format(package), end="")
54
55 # Match package names from x86_64, excluding 32-bit ones
56 package_url = re.findall(
57 r"\"({}-(?!32bit-)(?:[0-9a-z.%B]+)-(?:[0-9]+)\.(?:[0-9]+)\.x86_64.rpm)\"".format(
58 package
59 ),
60 contents,
61 )
62 arch = "x86_64"
63
64 # Fall back to noarch, place of packages without ELF binary
65 if len(package_url) == 0:
66 package_url = re.findall(
67 r"\"({}-(?:[0-9a-z.%B]+)-(?:[0-9]+)\.(?:[0-9]+)\.noarch.rpm)\"".format(
68 package
69 ),
70 contents_fallback,
71 )
72 arch = "noarch"
73
74 if len(package_url) != 1:
75 print()
76 print(
77 "Error: Could not find URL for '{}'.".format(package), file=sys.stderr
78 )
79 print(
80 "If this is a permanent error, please remove the package from the file packages.txt.",
81 file=sys.stderr,
82 )
83 continue
84 else:
85 package_url = package_url[0]
86
87 package_url = (
88 "https://ftp.gwdg.de/pub/opensuse/tumbleweed/repo/oss/{}/{}".format(
89 arch, package_url
90 )
91 )
92 package_filename = re.split(r"/", package_url)[-1]
93
94 # If the latest download is already available, skip this package
95 if package in package_data:
96 if package_data[package]["filename"] == package_filename:
97 print("current.")
98 continue
99
100 # Store information about downloaded package
101 package_data[package] = {
102 "filename": package_filename,
103 "needs_templates": True,
104 "needs_update": True,
105 }
106
107 # Prepare for downloading the file
108 package_dir = "downloads/{}".format(package)
109 local_filename = "downloads/{}/{}".format(package, package_filename)
110
111 # Ensure there are no leftovers of previous packages
112 os.makedirs(package_dir, exist_ok=True)
113 shutil.rmtree(package_dir)
114 os.makedirs(package_dir, exist_ok=True)
115
116 # Download new package
117 urllib.request.urlretrieve(package_url, local_filename)
118 print("downloaded.")
119
120 # Get the exact filename of data.tar with the extension
121 os.makedirs("{}/tmp".format(package_dir), exist_ok=True)
122
123 # Extract the package into a temporary file
124 with open("{}/tmp/tmpcpio".format(package_dir), "w") as outfile:
125 subprocess.run(["rpm2cpio", local_filename], stdout=outfile)
126 with open("{}/tmp/tmpcpio".format(package_dir), "r") as infile:
127 subprocess.run(
128 ["bsdcpio", "--quiet", "-i", "--make-directories"],
129 cwd="{}/tmp".format(package_dir),
130 stdin=infile,
131 )
132
133 # Move the manpages up
134 manpath = "{}/tmp/usr/share/man".format(package_dir)
135 for mandir in os.listdir(manpath):
136 if not mandir.startswith("man"):
137 continue
138 # Make sure the directory exists
139 os.makedirs(mandir, exist_ok=True)
140 shutil.move("{}/{}".format(manpath, mandir), "{}".format(package_dir))
141
142 # Cleanups
143 shutil.rmtree("downloads/{}/tmp".format(package))
144 os.remove(local_filename)
145
146 # Save package information in a JSON file
147 with open("downloads/packages.json", "w") as output_file:
148 output_file.write(json.dumps(package_data, sort_keys=True, indent=4))