Debian Codegraph @

src/mini-buildd-super-portext

raw
1 #!/usr/bin/python3
2 import fnmatch
3 import logging
4 import os
5 import urllib.request
6
7 import apt
8 import apt_pkg
9 import debian.deb822
10 import debian.debian_support
11
12 from mini_buildd import api, cli, client, events
13
14 LOG = logging.getLogger("mini_buildd")
15
16 #: Needed for man page hack in setup.py
17 DESCRIPTION = "EXPERIMENTAL: Portext a source package including all dependencies."
18 EPILOG = f"""
19 {DESCRIPTION}
20
21 Intended to help porting larger 'projects' (f.e. desktop environments) with
22 a lot of source packages and inter-package build dependencies.
23
24 This still has a lot of rough edges. Use with care.
25
26 This tool uses the apt cache of the system it is run on. This
27 needs to be set up pior to running. For example:
28
29 1. Run this in on system you want to PORT TO (f.e..: buster, 'deb' apt lines for buster).
30 2. Add 'deb-src' apt lines for the system you want to PORT FROM (f.e.: bullseye, 'deb-src' apt lines for bullseye).
31 3. sudo apt-get update
32 4. mini-buildd-super-portext -v http://admin@localhost:8066 http://localhost:3142/debian//pool/main/k/kguiaddons/kguiaddons_5.70.0-2.dsc buster-test-unstable
33
34 Per default, this will just print a list of DSC URLs it would portext
35 (in this order). Use --upload to actually run 'the portexts'.
36 """
37
38
39 class UniqueList(list):
40 def append(self, item):
41 if item not in self:
42 super().append(item)
43
44
45 class CLI(cli.CLI):
46 @classmethod
47 def _get_build_deps(cls, dsc_url):
48 def get_deps(key):
49 value = dsc.get(key)
50 LOG.debug("%s: %s", key, value)
51 return value.split(",") if value else []
52
53 with urllib.request.urlopen(dsc_url) as response:
54 dsc = debian.deb822.Dsc(response)
55 return [b.lstrip() for b in get_deps("Build-Depends") + get_deps("Build-Depends-Indep")]
56
57 class BuildDep():
58 """
59 A BuildDep object
60
61 >>> p = CLI.BuildDep("package")
62 >>> p.source, p.relation, p.version
63 ('package', '', '')
64
65 >>> p = CLI.BuildDep("package (> 1.2.3)")
66 >>> p.source, p.relation, p.version
67 ('package', '>', '1.2.3')
68
69 >>> p = CLI.BuildDep("package (> 1.2.3) [!kfreebsd]")
70 >>> p.source, p.relation, p.version
71 ('package', '>', '1.2.3')
72
73 >>> p = CLI.BuildDep("package [kfreebsd]")
74 >>> p.source, p.relation, p.version, p.arch_option, p.os_ignore
75 ('package', '', '', 'kfreebsd', True)
76
77 >>> p = CLI.BuildDep("package [!kfreebsd]")
78 >>> p.source, p.relation, p.version, p.arch_option, p.os_ignore
79 ('package', '', '', '!kfreebsd', True)
80
81 >>> p = CLI.BuildDep("package [linux]")
82 >>> p.source, p.relation, p.version, p.arch_option, p.os_ignore
83 ('package', '', '', 'linux', False)
84 """
85
86 def __init__(self, dep):
87 self.dep = dep
88 p0 = dep.partition(" ")
89 self.source = p0[0]
90
91 self.relation = ""
92 self.version = ""
93 if dep.find("(") != -1:
94 p1 = dep[dep.find("(") + 1:dep.find(")")].partition(" ")
95 self.relation = p1[0]
96 self.version = p1[2]
97
98 self.arch_option = ""
99 self.os_ignore = False
100 if dep.find("[") != -1:
101 self.arch_option = dep[dep.find("[") + 1:dep.find("]")]
102 self.os_ignore = (self.arch_option[0] == "!" and self.arch_option[1:].startswith("linux")) or (not self.arch_option.startswith("linux") and not self.arch_option.startswith("any"))
103
104 def __str__(self):
105 return f"{self.dep}"
106
107 @classmethod
108 def get_pool_path(cls, src_pkg, section="main"):
109 """Return the path in the pool where the files would be installed"""
110 if src_pkg.startswith('lib'):
111 subdir = src_pkg[:4]
112 else:
113 subdir = src_pkg[0]
114
115 return f"pool/{section}/{subdir}/{src_pkg}"
116
117 def make_dsc_url(self, source, version):
118 return self.mbd_mirror + f"/{self.get_pool_path(source)}/{source}_{version.rpartition(':')[2]}.dsc"
119
120 def lookup_pkg(self, pkg):
121 if self.apt_cache.is_virtual_package(pkg):
122 return self.apt_cache.get_providing_packages(pkg)[0]
123 return self.apt_cache[pkg]
124
125 @classmethod
126 def satisfied_pkg(cls, pkg, version):
127 LOG.debug("Checking: '%s' needs version >= '%s'", pkg.name, version)
128 for v in pkg.versions:
129 LOG.debug("Found: %s", v)
130 if debian.debian_support.Version(v.version) >= debian.debian_support.Version(version):
131 return True
132 return False
133
134 @classmethod
135 def lookup_src_pkg(cls, pkg):
136 package = None
137 version = None
138 rec = apt_pkg.SourceRecords()
139 while rec.lookup(pkg):
140 LOG.debug("lookup: %s %s", rec.package, rec.version)
141 v = debian.debian_support.Version(rec.version)
142 version = v if not version else max(version, v)
143 package = rec.package
144 return (package, f"{version}")
145
146 def get_unsatisified_build_deps(self, dsc_url):
147 LOG.info("Get unsatisfied build deps for: %s", dsc_url)
148
149 # Avoids endless loops with ring dependencies
150 if dsc_url in self.already_checked:
151 return
152 self.already_checked.append(dsc_url)
153
154 for d in self._get_build_deps(dsc_url):
155 LOG.info("Checking dep: %s", d)
156
157 dep = self.BuildDep(d)
158 if dep.os_ignore:
159 LOG.info("Ignoring dep, not for linux: %s", dep)
160 break
161
162 satisfied = False
163 pkg = None
164 pkg_name = dep.source
165 try:
166 pkg = self.lookup_pkg(dep.source)
167 pkg_name = pkg.name
168 if not dep.version:
169 satisfied = True
170 else:
171 satisfied = self.satisfied_pkg(pkg, dep.version)
172 except BaseException:
173 satisfied = False
174
175 if not satisfied:
176 source, version = self.lookup_src_pkg(pkg_name)
177 if not source:
178 LOG.error("Can't find source package for: %s", pkg_name)
179 elif source and dep.version and debian.debian_support.Version(version) < debian.debian_support.Version(dep.version):
180 LOG.error("Can't satisfy build dep: %s", dep)
181 else:
182 url = self.make_dsc_url(source, version)
183 self.get_unsatisified_build_deps(url)
184 self.portext.append(url)
185 else:
186 LOG.debug("Satisfied: %s", dep)
187
188 @classmethod
189 def uniq(cls, seq):
190 seen = set()
191 seen_add = seen.add
192 return [x for x in seq if x not in seen and not seen_add(x)]
193
194 class DscUrl():
195 def __init__(self, url):
196 self.url = url
197 p0 = os.path.basename(url).partition("_")
198 self.source = p0[0]
199 #: FIXME: We won't get epochs here
200 self.version = p0[2][:-4]
201
202 def is_satisfied(self, lookup_pkg):
203 try:
204 #: FIXME: This should rather look up for the source package name, and compare stable/testing source versions
205 pkg = lookup_pkg(self.source)
206 return CLI.satisfied_pkg(pkg, self.version)
207 except BaseException:
208 return False
209
210 def is_in_repo(self, clnt, distribution):
211 try:
212 clnt.api("find", {"source": self.source, "minimal_version": f"{self.version}~", "distributions": distribution})
213 return True
214 except BaseException:
215 return False
216
217 def wait_for_repo(self, clnt, distribution):
218 event = clnt.event(types=[events.Type.INSTALLED, events.Type.FAILED], source=self.source, minimal_version=f"{self.version}~", distribution=distribution)
219 if event.type == events.Type.INSTALLED:
220 LOG.info("Package result: %s", event)
221 LOG.error("Package result: %s", event)
222
223 def run_url(self, dsc_url):
224 LOG.info("RUN_URL: %s", dsc_url)
225
226 dsc = self.DscUrl(dsc_url)
227 if dsc.is_satisfied(self.lookup_pkg):
228 LOG.info("Already satisfied, skipping: %s", dsc.url)
229 return
230
231 if dsc.is_in_repo(self.mbd_client, self.mbd_distribution):
232 LOG.info("Already in repo, skipping: %s", dsc.url)
233 return
234
235 self.get_unsatisified_build_deps(dsc.url)
236 self.portext.append(dsc_url)
237
238 def run_pattern(self, pattern):
239 class Filter(apt.cache.Filter):
240 def apply(self, pkg):
241 return fnmatch.fnmatch(pkg.name, pattern)
242
243 filtered = apt.cache.FilteredCache(self.apt_cache)
244 filtered.set_filter(Filter())
245
246 src_packages = self.uniq([p.candidate.source_name for p in filtered])
247 for p in src_packages:
248 package, version = self.lookup_src_pkg(p)
249 if package:
250 self.run_url(self.make_dsc_url(package, version))
251 else:
252 LOG.warning("No source package found: %s", p)
253
254 def __init__(self):
255 super().__init__("mbu-super-portext", DESCRIPTION, epilog=EPILOG)
256 self._add_endpoint(self.parser)
257 self.parser.add_argument("source", action="store", metavar="SOURCE", help="Pattern or DSC URL")
258 self.parser.add_argument("distribution", action="store", metavar="DISTRIBUTION", help="distribution to portext to")
259 self.parser.add_argument("-M", "--mirror", action="store", metavar="MIRROR", default="http://localhost:3142/debian/", help="Debian mirror to use")
260 self.parser.add_argument("-U", "--upload", action="store_true", help="Non-dry run: Actually upload packages")
261
262 apt_pkg.init()
263 self.apt_cache = apt.Cache()
264 self.mbd_mirror = None
265 self.mbd_distribution = None
266 self.mbd_client = None
267
268 self.already_checked = UniqueList()
269 self.portext = UniqueList()
270
271 def runcli(self):
272 # local setup
273 self.mbd_mirror = self.args.mirror
274 self.mbd_distribution = self.args.distribution
275 self.mbd_client = client.Client(self.args.endpoint)
276
277 if self.args.source.find("://") == -1:
278 self.run_pattern(self.args.source)
279 else:
280 self.run_url(self.args.source)
281
282 if self.args.upload:
283 for url in self.portext:
284 self.mbd_client.api(api.PortExt(dsc=url, distributions=self.mbd_distribution))
285 self.DscUrl(url).wait_for_repo(self.mbd_client, self.mbd_distribution)
286 else:
287 LOG.info("Ordered portext list:")
288 for url in self.portext:
289 print(url)
290
291
292 if __name__ == "__main__":
293 CLI().run()