Debian Codegraph @

spark/runners/debspawn.py

raw
1 # -*- coding: utf-8 -*-
2 #
3 # Copyright (c) 2012-2013 Paul Tagliamonte <[email protected]>
4 # Copyright (c) 2016-2018 Matthias Klumpp <[email protected]>
5 #
6 # Permission is hereby granted, free of charge, to any person obtaining a
7 # copy of this software and associated documentation files (the "Software"),
8 # to deal in the Software without restriction, including without limitation
9 # the rights to use, copy, modify, merge, publish, distribute, sublicense,
10 # and/or sell copies of the Software, and to permit persons to whom the
11 # Software is furnished to do so, subject to the following conditions:
12 #
13 # The above copyright notice and this permission notice shall be included in
14 # all copies or substantial portions of the Software.
15 #
16 # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
17 # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
18 # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
19 # THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
20 # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
21 # FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
22 # DEALINGS IN THE SOFTWARE.
23
24 import os
25 import re
26 import glob
27 from io import StringIO
28 from datetime import timedelta
29
30 import firehose.parsers.gcc as fgcc
31 from firehose.model import Stats, Analysis
32
33 from spark.utils import RunnerError, RunnerResult
34 from spark.utils.command import safe_run, run_logged, run_command
35 from spark.utils.firehose import create_firehose
36
37 STATS = re.compile('Build needed (?P<time>.*), (?P<space>.*) dis(c|k) space')
38
39
40 def parse_debspawn_log(log, sut):
41 gccversion = None
42 stats = None
43
44 for line in log.splitlines():
45 flag = 'Toolchain package versions: '
46 stat = STATS.match(line)
47 if stat:
48 info = stat.groupdict()
49 hours, minutes, seconds = [int(x) for x in info['time'].split(':')]
50 timed = timedelta(hours=hours, minutes=minutes, seconds=seconds)
51 stats = Stats(timed.total_seconds())
52 if line.startswith(flag):
53 line = line[len(flag) :].strip()
54 packages = line.split(' ')
55 versions = {}
56 for package in packages:
57 if '_' not in package:
58 continue
59 b, bv = package.split('_', 1)
60 versions[b] = bv
61 vs = list(filter(lambda x: x.startswith('gcc'), versions))
62 if vs == []:
63 continue
64 vs = vs[0]
65 gccversion = versions[vs]
66
67 obj = fgcc.parse_file(StringIO(log), sut=sut, gccversion=gccversion, stats=stats)
68
69 return obj
70
71
72 def debspawn_build(
73 jlog, dsc, maintainer, suite, affinity, build_arch, build_indep, analysis: Analysis
74 ) -> tuple[Analysis, str, bool, bool, list[str] | None]:
75 if not dsc.endswith('.dsc'):
76 raise ValueError('WTF')
77
78 ds_cmd = [
79 'debspawn',
80 'build',
81 '--no-buildlog',
82 '--arch={affinity}'.format(affinity=affinity),
83 '--results-dir={cwd}'.format(cwd=os.getcwd()),
84 ]
85
86 if build_arch and not build_indep:
87 ds_cmd.append('--only=arch')
88 elif not build_arch and build_indep:
89 ds_cmd.append('--only=indep')
90 else:
91 ds_cmd.append('--only=binary')
92 if maintainer:
93 ds_cmd.append('--maintainer={maintainer}'.format(maintainer=maintainer))
94 ds_cmd.append(suite)
95 ds_cmd.append(dsc)
96
97 ret, out = run_logged(jlog, ds_cmd, True)
98 for line in out.splitlines():
99 if ret != 0:
100 if line.startswith('ERROR: The container image for'):
101 # We likely have a missing container image for this build type
102 raise RunnerError('This worker is missing an environment: {}'.format(line))
103 elif 'The following packages have unmet dependencies:' in line:
104 # We are waiting for dependencies
105 return (analysis, out, True, True, None)
106
107 ftbfs = ret != 0
108 base, _ = os.path.basename(dsc).rsplit('.', 1)
109 changes = glob.glob('{base}_*.changes'.format(base=base))
110
111 return (analysis, out, ftbfs, False, changes)
112
113
114 def checkout(dsc_url):
115 safe_run(['dget', '-u', '-d', dsc_url])
116 return os.path.basename(dsc_url)
117
118
119 def get_version():
120 out, _, ret = run_command(['debspawn', '--version'])
121 if ret != 0:
122 raise RuntimeError('debspawn is not installed')
123 return ('debspawn', out.strip())
124
125
126 def run(
127 jlog, job, jdata
128 ) -> tuple[RunnerResult, list[os.PathLike | str] | None, os.PathLike | str | None]:
129 arch_name = job['architecture']
130 build_arch = arch_name != 'all'
131 build_indep = arch_name == 'all' or jdata['do_indep']
132 maintainer = jdata.get('maintainer')
133
134 firehose = create_firehose(
135 'source', jdata['package_name'], jdata['package_version'], build_arch, get_version
136 )
137
138 dsc = checkout(jdata['dsc_url'])
139 (
140 firehose,
141 out,
142 ftbfs,
143 depwait,
144 changes_list,
145 ) = debspawn_build(
146 jlog, dsc, maintainer, jdata['suite'], arch_name, build_arch, build_indep, firehose
147 )
148
149 if not changes_list and not ftbfs:
150 print(out)
151 print(changes_list)
152 print(list(glob.glob('*')))
153 raise RunnerError('Um. No changes but no FTBFS.')
154
155 changes: str | None = None
156 if not ftbfs:
157 changes = os.path.join(os.getcwd(), changes_list[0])
158
159 _, _, v = jdata['package_version'].rpartition(':')
160 prefix = '%s_%s_%s.%s' % (jdata['package_name'], v, arch_name, job['uuid'])
161 firehose_fname = '{prefix}.firehose.xml'.format(prefix=prefix)
162
163 files: list[os.PathLike | str] = []
164 with open(firehose_fname, 'wb') as fd:
165 fd.write(firehose.to_xml_bytes())
166 files.append(os.path.abspath(firehose_fname))
167
168 result = RunnerResult.SUCCESS
169 if depwait:
170 result = RunnerResult.DEPWAIT
171 elif ftbfs:
172 result = RunnerResult.FAILURE
173
174 return result, files, changes