| 1 |
#!/usr/bin/python3 |
| 2 |
import fnmatch |
| 3 |
import logging |
| 4 |
import os |
| 5 |
import urllib.request |
| 6 |
|
| 7 |
import apt |
| 8 |
import apt_pkg |
| 9 |
import debian.deb822 |
| 10 |
import debian.debian_support |
| 11 |
|
| 12 |
from mini_buildd import api, cli, client, events |
| 13 |
|
| 14 |
LOG = logging.getLogger("mini_buildd") |
| 15 |
|
| 16 |
#: Needed for man page hack in setup.py |
| 17 |
DESCRIPTION = "EXPERIMENTAL: Portext a source package including all dependencies." |
| 18 |
EPILOG = f""" |
| 19 |
{DESCRIPTION} |
| 20 |
|
| 21 |
Intended to help porting larger 'projects' (f.e. desktop environments) with |
| 22 |
a lot of source packages and inter-package build dependencies. |
| 23 |
|
| 24 |
This still has a lot of rough edges. Use with care. |
| 25 |
|
| 26 |
This tool uses the apt cache of the system it is run on. This |
| 27 |
needs to be set up pior to running. For example: |
| 28 |
|
| 29 |
1. Run this in on system you want to PORT TO (f.e..: buster, 'deb' apt lines for buster). |
| 30 |
2. Add 'deb-src' apt lines for the system you want to PORT FROM (f.e.: bullseye, 'deb-src' apt lines for bullseye). |
| 31 |
3. sudo apt-get update |
| 32 |
4. mini-buildd-super-portext -v http://admin@localhost:8066 http://localhost:3142/debian//pool/main/k/kguiaddons/kguiaddons_5.70.0-2.dsc buster-test-unstable |
| 33 |
|
| 34 |
Per default, this will just print a list of DSC URLs it would portext |
| 35 |
(in this order). Use --upload to actually run 'the portexts'. |
| 36 |
""" |
| 37 |
|
| 38 |
|
| 39 |
class UniqueList(list): |
| 40 |
def append(self, item): |
| 41 |
if item not in self: |
| 42 |
super().append(item) |
| 43 |
|
| 44 |
|
| 45 |
class CLI(cli.CLI): |
| 46 |
@classmethod |
| 47 |
def _get_build_deps(cls, dsc_url): |
| 48 |
def get_deps(key): |
| 49 |
value = dsc.get(key) |
| 50 |
LOG.debug("%s: %s", key, value) |
| 51 |
return value.split(",") if value else [] |
| 52 |
|
| 53 |
with urllib.request.urlopen(dsc_url) as response: |
| 54 |
dsc = debian.deb822.Dsc(response) |
| 55 |
return [b.lstrip() for b in get_deps("Build-Depends") + get_deps("Build-Depends-Indep")] |
| 56 |
|
| 57 |
class BuildDep(): |
| 58 |
""" |
| 59 |
A BuildDep object |
| 60 |
|
| 61 |
>>> p = CLI.BuildDep("package") |
| 62 |
>>> p.source, p.relation, p.version |
| 63 |
('package', '', '') |
| 64 |
|
| 65 |
>>> p = CLI.BuildDep("package (> 1.2.3)") |
| 66 |
>>> p.source, p.relation, p.version |
| 67 |
('package', '>', '1.2.3') |
| 68 |
|
| 69 |
>>> p = CLI.BuildDep("package (> 1.2.3) [!kfreebsd]") |
| 70 |
>>> p.source, p.relation, p.version |
| 71 |
('package', '>', '1.2.3') |
| 72 |
|
| 73 |
>>> p = CLI.BuildDep("package [kfreebsd]") |
| 74 |
>>> p.source, p.relation, p.version, p.arch_option, p.os_ignore |
| 75 |
('package', '', '', 'kfreebsd', True) |
| 76 |
|
| 77 |
>>> p = CLI.BuildDep("package [!kfreebsd]") |
| 78 |
>>> p.source, p.relation, p.version, p.arch_option, p.os_ignore |
| 79 |
('package', '', '', '!kfreebsd', True) |
| 80 |
|
| 81 |
>>> p = CLI.BuildDep("package [linux]") |
| 82 |
>>> p.source, p.relation, p.version, p.arch_option, p.os_ignore |
| 83 |
('package', '', '', 'linux', False) |
| 84 |
""" |
| 85 |
|
| 86 |
def __init__(self, dep): |
| 87 |
self.dep = dep |
| 88 |
p0 = dep.partition(" ") |
| 89 |
self.source = p0[0] |
| 90 |
|
| 91 |
self.relation = "" |
| 92 |
self.version = "" |
| 93 |
if dep.find("(") != -1: |
| 94 |
p1 = dep[dep.find("(") + 1:dep.find(")")].partition(" ") |
| 95 |
self.relation = p1[0] |
| 96 |
self.version = p1[2] |
| 97 |
|
| 98 |
self.arch_option = "" |
| 99 |
self.os_ignore = False |
| 100 |
if dep.find("[") != -1: |
| 101 |
self.arch_option = dep[dep.find("[") + 1:dep.find("]")] |
| 102 |
self.os_ignore = (self.arch_option[0] == "!" and self.arch_option[1:].startswith("linux")) or (not self.arch_option.startswith("linux") and not self.arch_option.startswith("any")) |
| 103 |
|
| 104 |
def __str__(self): |
| 105 |
return f"{self.dep}" |
| 106 |
|
| 107 |
@classmethod |
| 108 |
def get_pool_path(cls, src_pkg, section="main"): |
| 109 |
"""Return the path in the pool where the files would be installed""" |
| 110 |
if src_pkg.startswith('lib'): |
| 111 |
subdir = src_pkg[:4] |
| 112 |
else: |
| 113 |
subdir = src_pkg[0] |
| 114 |
|
| 115 |
return f"pool/{section}/{subdir}/{src_pkg}" |
| 116 |
|
| 117 |
def make_dsc_url(self, source, version): |
| 118 |
return self.mbd_mirror + f"/{self.get_pool_path(source)}/{source}_{version.rpartition(':')[2]}.dsc" |
| 119 |
|
| 120 |
def lookup_pkg(self, pkg): |
| 121 |
if self.apt_cache.is_virtual_package(pkg): |
| 122 |
return self.apt_cache.get_providing_packages(pkg)[0] |
| 123 |
return self.apt_cache[pkg] |
| 124 |
|
| 125 |
@classmethod |
| 126 |
def satisfied_pkg(cls, pkg, version): |
| 127 |
LOG.debug("Checking: '%s' needs version >= '%s'", pkg.name, version) |
| 128 |
for v in pkg.versions: |
| 129 |
LOG.debug("Found: %s", v) |
| 130 |
if debian.debian_support.Version(v.version) >= debian.debian_support.Version(version): |
| 131 |
return True |
| 132 |
return False |
| 133 |
|
| 134 |
@classmethod |
| 135 |
def lookup_src_pkg(cls, pkg): |
| 136 |
package = None |
| 137 |
version = None |
| 138 |
rec = apt_pkg.SourceRecords() |
| 139 |
while rec.lookup(pkg): |
| 140 |
LOG.debug("lookup: %s %s", rec.package, rec.version) |
| 141 |
v = debian.debian_support.Version(rec.version) |
| 142 |
version = v if not version else max(version, v) |
| 143 |
package = rec.package |
| 144 |
return (package, f"{version}") |
| 145 |
|
| 146 |
def get_unsatisified_build_deps(self, dsc_url): |
| 147 |
LOG.info("Get unsatisfied build deps for: %s", dsc_url) |
| 148 |
|
| 149 |
# Avoids endless loops with ring dependencies |
| 150 |
if dsc_url in self.already_checked: |
| 151 |
return |
| 152 |
self.already_checked.append(dsc_url) |
| 153 |
|
| 154 |
for d in self._get_build_deps(dsc_url): |
| 155 |
LOG.info("Checking dep: %s", d) |
| 156 |
|
| 157 |
dep = self.BuildDep(d) |
| 158 |
if dep.os_ignore: |
| 159 |
LOG.info("Ignoring dep, not for linux: %s", dep) |
| 160 |
break |
| 161 |
|
| 162 |
satisfied = False |
| 163 |
pkg = None |
| 164 |
pkg_name = dep.source |
| 165 |
try: |
| 166 |
pkg = self.lookup_pkg(dep.source) |
| 167 |
pkg_name = pkg.name |
| 168 |
if not dep.version: |
| 169 |
satisfied = True |
| 170 |
else: |
| 171 |
satisfied = self.satisfied_pkg(pkg, dep.version) |
| 172 |
except BaseException: |
| 173 |
satisfied = False |
| 174 |
|
| 175 |
if not satisfied: |
| 176 |
source, version = self.lookup_src_pkg(pkg_name) |
| 177 |
if not source: |
| 178 |
LOG.error("Can't find source package for: %s", pkg_name) |
| 179 |
elif source and dep.version and debian.debian_support.Version(version) < debian.debian_support.Version(dep.version): |
| 180 |
LOG.error("Can't satisfy build dep: %s", dep) |
| 181 |
else: |
| 182 |
url = self.make_dsc_url(source, version) |
| 183 |
self.get_unsatisified_build_deps(url) |
| 184 |
self.portext.append(url) |
| 185 |
else: |
| 186 |
LOG.debug("Satisfied: %s", dep) |
| 187 |
|
| 188 |
@classmethod |
| 189 |
def uniq(cls, seq): |
| 190 |
seen = set() |
| 191 |
seen_add = seen.add |
| 192 |
return [x for x in seq if x not in seen and not seen_add(x)] |
| 193 |
|
| 194 |
class DscUrl(): |
| 195 |
def __init__(self, url): |
| 196 |
self.url = url |
| 197 |
p0 = os.path.basename(url).partition("_") |
| 198 |
self.source = p0[0] |
| 199 |
#: FIXME: We won't get epochs here |
| 200 |
self.version = p0[2][:-4] |
| 201 |
|
| 202 |
def is_satisfied(self, lookup_pkg): |
| 203 |
try: |
| 204 |
#: FIXME: This should rather look up for the source package name, and compare stable/testing source versions |
| 205 |
pkg = lookup_pkg(self.source) |
| 206 |
return CLI.satisfied_pkg(pkg, self.version) |
| 207 |
except BaseException: |
| 208 |
return False |
| 209 |
|
| 210 |
def is_in_repo(self, clnt, distribution): |
| 211 |
try: |
| 212 |
clnt.api("find", {"source": self.source, "minimal_version": f"{self.version}~", "distributions": distribution}) |
| 213 |
return True |
| 214 |
except BaseException: |
| 215 |
return False |
| 216 |
|
| 217 |
def wait_for_repo(self, clnt, distribution): |
| 218 |
event = clnt.event(types=[events.Type.INSTALLED, events.Type.FAILED], source=self.source, minimal_version=f"{self.version}~", distribution=distribution) |
| 219 |
if event.type == events.Type.INSTALLED: |
| 220 |
LOG.info("Package result: %s", event) |
| 221 |
LOG.error("Package result: %s", event) |
| 222 |
|
| 223 |
def run_url(self, dsc_url): |
| 224 |
LOG.info("RUN_URL: %s", dsc_url) |
| 225 |
|
| 226 |
dsc = self.DscUrl(dsc_url) |
| 227 |
if dsc.is_satisfied(self.lookup_pkg): |
| 228 |
LOG.info("Already satisfied, skipping: %s", dsc.url) |
| 229 |
return |
| 230 |
|
| 231 |
if dsc.is_in_repo(self.mbd_client, self.mbd_distribution): |
| 232 |
LOG.info("Already in repo, skipping: %s", dsc.url) |
| 233 |
return |
| 234 |
|
| 235 |
self.get_unsatisified_build_deps(dsc.url) |
| 236 |
self.portext.append(dsc_url) |
| 237 |
|
| 238 |
def run_pattern(self, pattern): |
| 239 |
class Filter(apt.cache.Filter): |
| 240 |
def apply(self, pkg): |
| 241 |
return fnmatch.fnmatch(pkg.name, pattern) |
| 242 |
|
| 243 |
filtered = apt.cache.FilteredCache(self.apt_cache) |
| 244 |
filtered.set_filter(Filter()) |
| 245 |
|
| 246 |
src_packages = self.uniq([p.candidate.source_name for p in filtered]) |
| 247 |
for p in src_packages: |
| 248 |
package, version = self.lookup_src_pkg(p) |
| 249 |
if package: |
| 250 |
self.run_url(self.make_dsc_url(package, version)) |
| 251 |
else: |
| 252 |
LOG.warning("No source package found: %s", p) |
| 253 |
|
| 254 |
def __init__(self): |
| 255 |
super().__init__("mbu-super-portext", DESCRIPTION, epilog=EPILOG) |
| 256 |
self._add_endpoint(self.parser) |
| 257 |
self.parser.add_argument("source", action="store", metavar="SOURCE", help="Pattern or DSC URL") |
| 258 |
self.parser.add_argument("distribution", action="store", metavar="DISTRIBUTION", help="distribution to portext to") |
| 259 |
self.parser.add_argument("-M", "--mirror", action="store", metavar="MIRROR", default="http://localhost:3142/debian/", help="Debian mirror to use") |
| 260 |
self.parser.add_argument("-U", "--upload", action="store_true", help="Non-dry run: Actually upload packages") |
| 261 |
|
| 262 |
apt_pkg.init() |
| 263 |
self.apt_cache = apt.Cache() |
| 264 |
self.mbd_mirror = None |
| 265 |
self.mbd_distribution = None |
| 266 |
self.mbd_client = None |
| 267 |
|
| 268 |
self.already_checked = UniqueList() |
| 269 |
self.portext = UniqueList() |
| 270 |
|
| 271 |
def runcli(self): |
| 272 |
# local setup |
| 273 |
self.mbd_mirror = self.args.mirror |
| 274 |
self.mbd_distribution = self.args.distribution |
| 275 |
self.mbd_client = client.Client(self.args.endpoint) |
| 276 |
|
| 277 |
if self.args.source.find("://") == -1: |
| 278 |
self.run_pattern(self.args.source) |
| 279 |
else: |
| 280 |
self.run_url(self.args.source) |
| 281 |
|
| 282 |
if self.args.upload: |
| 283 |
for url in self.portext: |
| 284 |
self.mbd_client.api(api.PortExt(dsc=url, distributions=self.mbd_distribution)) |
| 285 |
self.DscUrl(url).wait_for_repo(self.mbd_client, self.mbd_distribution) |
| 286 |
else: |
| 287 |
LOG.info("Ordered portext list:") |
| 288 |
for url in self.portext: |
| 289 |
print(url) |
| 290 |
|
| 291 |
|
| 292 |
if __name__ == "__main__": |
| 293 |
CLI().run() |