| 1 |
#!/usr/bin/python3 |
| 2 |
# vi: expandtab:sts=4 |
| 3 |
|
| 4 |
# Copyright (C) 2011 Jelmer Vernooij <[email protected]> |
| 5 |
# |
| 6 |
# ################################################################## |
| 7 |
# |
| 8 |
# This program is free software; you can redistribute it and/or |
| 9 |
# modify it under the terms of the GNU General Public License |
| 10 |
# as published by the Free Software Foundation; version 3. |
| 11 |
# |
| 12 |
# This program is distributed in the hope that it will be useful, |
| 13 |
# but WITHOUT ANY WARRANTY; without even the implied warranty of |
| 14 |
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the |
| 15 |
# GNU General Public License for more details. |
| 16 |
# |
| 17 |
# See file /usr/share/common-licenses/GPL-3 for more details. |
| 18 |
# |
| 19 |
# ################################################################## |
| 20 |
# |
| 21 |
|
| 22 |
"""Show the status of the recipes owned by a particular user. |
| 23 |
""" |
| 24 |
|
| 25 |
from io import StringIO |
| 26 |
import gzip |
| 27 |
import optparse |
| 28 |
import os |
| 29 |
import re |
| 30 |
import sys |
| 31 |
import urllib.request, urllib.parse, urllib.error |
| 32 |
|
| 33 |
from lptools import config |
| 34 |
|
| 35 |
try: |
| 36 |
import tdb |
| 37 |
except ImportError: |
| 38 |
cache = {} |
| 39 |
else: |
| 40 |
cache = tdb.Tdb("recipe-status-cache.tdb", 1000, tdb.DEFAULT, |
| 41 |
os.O_RDWR|os.O_CREAT) |
| 42 |
|
| 43 |
def gather_per_distroseries_source_builds(recipe): |
| 44 |
last_per_distroseries = {} |
| 45 |
for build in recipe.completed_builds: |
| 46 |
if build.distro_series.self_link not in recipe.distroseries: |
| 47 |
# Skip distro series that are no longer being build |
| 48 |
continue |
| 49 |
distro_series_name = build.distro_series.name |
| 50 |
previous_build = last_per_distroseries.get(distro_series_name) |
| 51 |
if previous_build is None or previous_build.datecreated < build.datecreated: |
| 52 |
last_per_distroseries[distro_series_name] = build |
| 53 |
return last_per_distroseries |
| 54 |
|
| 55 |
|
| 56 |
def build_failure_link(build): |
| 57 |
if build.buildstate == "Failed to upload": |
| 58 |
return build.upload_log_url |
| 59 |
elif build.buildstate in ("Failed to build", "Dependency wait", "Chroot problem"): |
| 60 |
return build.build_log_url |
| 61 |
else: |
| 62 |
return None |
| 63 |
|
| 64 |
version_matcher = re.compile("^dpkg-buildpackage: source version (.*)$") |
| 65 |
source_name_matcher = re.compile("^dpkg-buildpackage: source package (.*)$") |
| 66 |
|
| 67 |
|
| 68 |
def source_build_find_version(source_build): |
| 69 |
cached_version = cache.get("version/%s" % str(source_build.self_link)) |
| 70 |
if cached_version: |
| 71 |
return tuple(cached_version.split(" ")) |
| 72 |
# FIXME: Find a more efficient way to retrieve the package/version that was built |
| 73 |
build_log_gz = urllib.request.urlopen(source_build.build_log_url) |
| 74 |
build_log = gzip.GzipFile(fileobj=StringIO(build_log_gz.read())) |
| 75 |
version = None |
| 76 |
source_name = None |
| 77 |
for l in build_log.readlines(): |
| 78 |
m = version_matcher.match(l) |
| 79 |
if m: |
| 80 |
version = m.group(1) |
| 81 |
m = source_name_matcher.match(l) |
| 82 |
if m: |
| 83 |
source_name = m.group(1) |
| 84 |
if not source_name: |
| 85 |
raise Exception("unable to find source name in %s" % |
| 86 |
source_build.build_log_url) |
| 87 |
if not version: |
| 88 |
raise Exception("unable to find version in %s" % |
| 89 |
source_build.build_log_url) |
| 90 |
cache["version/%s" % str(source_build.self_link)] = "%s %s" % ( |
| 91 |
source_name, version) |
| 92 |
return (source_name, version) |
| 93 |
|
| 94 |
|
| 95 |
def find_binary_builds(recipe, source_builds): |
| 96 |
"""Gather binary builds for a set of source builds. |
| 97 |
|
| 98 |
:param recipe: Recipe to build |
| 99 |
:param source_builds: Source builds to analyse |
| 100 |
:return: Dictionary mapping series name to binary builds |
| 101 |
""" |
| 102 |
ret = {} |
| 103 |
for source_build in source_builds: |
| 104 |
archive = source_build.archive |
| 105 |
(source_name, version) = source_build_find_version(source_build) |
| 106 |
sources = archive.getPublishedSources( |
| 107 |
distro_series=source_build.distro_series, |
| 108 |
exact_match=True, pocket="Release", source_name=source_name, |
| 109 |
version=version) |
| 110 |
assert len(sources) == 1 |
| 111 |
source = sources[0] |
| 112 |
ret[source_build.distro_series.name] = source.getBuilds() |
| 113 |
return ret |
| 114 |
|
| 115 |
|
| 116 |
def build_failure_summary(build): |
| 117 |
# FIXME: Perhaps parse the relevant logs and extract a summary line? |
| 118 |
return build.buildstate |
| 119 |
|
| 120 |
|
| 121 |
def build_class(build): |
| 122 |
"""Determine the CSS class for a build status. |
| 123 |
|
| 124 |
:param build: Launchpad build object |
| 125 |
:return: CSS class name |
| 126 |
""" |
| 127 |
return { |
| 128 |
"Failed to build": "failed-to-build", |
| 129 |
"Failed to upload": "failed-to-upload", |
| 130 |
"Dependency wait": "dependency-wait", |
| 131 |
"Chroot problem": "chroot-problem", |
| 132 |
"Uploading build": "uploading-build", |
| 133 |
"Currently building": "currently-building", |
| 134 |
"Build for superseded Source": "superseded-source", |
| 135 |
"Successfully built": "successfully-built", |
| 136 |
"Needs building": "needs-building", |
| 137 |
}[build.buildstate] |
| 138 |
|
| 139 |
|
| 140 |
def filter_source_builds(builds): |
| 141 |
"""Filter out successful and failed builds. |
| 142 |
|
| 143 |
:param builds: List of builds |
| 144 |
:return: Tuple with set of successful and set of failed builds |
| 145 |
""" |
| 146 |
sp_success = set() |
| 147 |
sp_failures = set() |
| 148 |
for build in builds: |
| 149 |
if build.buildstate == "Successfully built": |
| 150 |
sp_success.add(build) |
| 151 |
else: |
| 152 |
sp_failures.add(build) |
| 153 |
return (sp_success, sp_failures) |
| 154 |
|
| 155 |
|
| 156 |
def recipe_status_html(launchpad, person, recipes, outf): |
| 157 |
"""Render a recipe status table in HTML. |
| 158 |
|
| 159 |
:param launchpad: launchpadlib Launchpad object |
| 160 |
:param person: Person owning all the recipes |
| 161 |
:param recipes: List of recipes to render |
| 162 |
:param outf: File-like object to write HTML to |
| 163 |
""" |
| 164 |
from chameleon.zpt.loader import TemplateLoader |
| 165 |
tl = TemplateLoader(os.path.join(config.data_dir(), "templates")) |
| 166 |
relevant_distroseries = set() |
| 167 |
source_builds = {} |
| 168 |
binary_builds = {} |
| 169 |
all_binary_builds_ok = {} |
| 170 |
for recipe in recipes: |
| 171 |
sys.stderr.write("Processing recipe %s\n" % recipe.name) |
| 172 |
last_per_distroseries = gather_per_distroseries_source_builds(recipe) |
| 173 |
source_builds[recipe.name] = last_per_distroseries |
| 174 |
relevant_distroseries.update(set(last_per_distroseries)) |
| 175 |
(sp_success, sp_failures) = filter_source_builds(list(last_per_distroseries.values())) |
| 176 |
binary_builds[recipe.name] = find_binary_builds(recipe, sp_success) |
| 177 |
all_binary_builds_ok[recipe.name] = {} |
| 178 |
for distroseries, recipe_binary_builds in binary_builds[recipe.name].items(): |
| 179 |
all_binary_builds_ok[recipe.name][distroseries] = all( |
| 180 |
[bb.buildstate == "Successfully built" for bb in recipe_binary_builds]) |
| 181 |
relevant_distroseries = list(relevant_distroseries) |
| 182 |
relevant_distroseries.sort() |
| 183 |
page = tl.load("recipe-status.html") |
| 184 |
outf.write(page.render(person=person, |
| 185 |
relevant_distroseries=relevant_distroseries, |
| 186 |
recipes=person.recipes, source_builds=source_builds, |
| 187 |
build_failure_summary=build_failure_summary, |
| 188 |
build_failure_link=build_failure_link, |
| 189 |
binary_builds=binary_builds, |
| 190 |
ubuntu=launchpad.distributions["ubuntu"], |
| 191 |
build_class=build_class, |
| 192 |
all_binary_builds_ok=all_binary_builds_ok)) |
| 193 |
|
| 194 |
|
| 195 |
def recipe_status_text(recipes, outf): |
| 196 |
"""Display a recipe status table in plain text. |
| 197 |
|
| 198 |
:param recipes: List of recipes to display status of |
| 199 |
:param outf: file-like object to write to |
| 200 |
""" |
| 201 |
for recipe in recipes: |
| 202 |
last_per_distroseries = gather_per_distroseries_source_builds(recipe) |
| 203 |
(sp_success, sp_failures) = filter_source_builds( |
| 204 |
list(last_per_distroseries.values())) |
| 205 |
sp_success_distroseries = [build.distro_series.name for build in sp_success] |
| 206 |
if sp_failures: |
| 207 |
outf.write("%s source build failures (%s successful):\n" % ( |
| 208 |
recipe.name, ", ".join(sp_success_distroseries))) |
| 209 |
for failed_build in sp_failures: |
| 210 |
url = build_failure_link(failed_build) |
| 211 |
outf.write(" %s(%s)" % ( |
| 212 |
failed_build.distro_series.name, failed_build.buildstate)) |
| 213 |
if url: |
| 214 |
outf.write(": %s" % url) |
| 215 |
outf.write("\n") |
| 216 |
elif sp_success: |
| 217 |
outf.write("%s source built successfully on %s\n" % ( |
| 218 |
recipe.name, ", ".join(sp_success_distroseries))) |
| 219 |
else: |
| 220 |
outf.write("%s never built\n" % recipe.name) |
| 221 |
binary_builds = find_binary_builds(recipe, sp_success) |
| 222 |
for source_build in sp_success: |
| 223 |
for binary_build in binary_builds.get(source_build, []): |
| 224 |
if binary_build.buildstate != "Successfully built": |
| 225 |
url = build_failure_link(binary_build) |
| 226 |
outf.write(" %s,%s(%s)" % |
| 227 |
(binary_build.distro_series.name, |
| 228 |
binary_build.arch_tag, |
| 229 |
binary_build.buildstate)) |
| 230 |
if url: |
| 231 |
outf.write(": %s" % url) |
| 232 |
outf.write("\n") |
| 233 |
|
| 234 |
|
| 235 |
def main(argv): |
| 236 |
parser = optparse.OptionParser('%prog [options] PERSON\n\n' |
| 237 |
' PERSON is the launchpad person or team whose recipes to check') |
| 238 |
parser.add_option("--html", help="Generate HTML", action="store_true") |
| 239 |
opts, args = parser.parse_args() |
| 240 |
if len(args) != 1: |
| 241 |
parser.print_usage() |
| 242 |
return 1 |
| 243 |
person = args[0] |
| 244 |
launchpad = config.get_launchpad("recipe-status") |
| 245 |
person = launchpad.people[person] |
| 246 |
|
| 247 |
if opts.html: |
| 248 |
recipe_status_html(launchpad, person, person.recipes, sys.stdout) |
| 249 |
else: |
| 250 |
recipe_status_text(person.recipes, sys.stdout) |
| 251 |
|
| 252 |
if __name__ == '__main__': |
| 253 |
sys.exit(main(sys.argv)) |