Debian Codegraph @

jenkins_jobs/builder.py

raw
1 #!/usr/bin/env python
2 # Copyright (C) 2012 OpenStack, LLC.
3 #
4 # Licensed under the Apache License, Version 2.0 (the "License");
5 # you may not use this file except in compliance with the License.
6 # You may obtain a copy of the License at
7 #
8 # http://www.apache.org/licenses/LICENSE-2.0
9 #
10 # Unless required by applicable law or agreed to in writing, software
11 # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
12 # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
13 # License for the specific language governing permissions and limitations
14 # under the License.
15
16 # Manage jobs in Jenkins server
17
18 import errno
19 import hashlib
20 import io
21 import logging
22 import os
23 from pprint import pformat
24 import re
25 from six.moves.urllib.parse import quote
26 import time
27 import xml.etree.ElementTree as XML
28
29 import jenkins
30
31 from jenkins_jobs.alphanum import AlphanumSort
32 from jenkins_jobs.cache import JobCache
33 from jenkins_jobs.constants import MAGIC_MANAGE_STRING
34 from jenkins_jobs.parallel import concurrent
35 from jenkins_jobs import utils
36
37 __all__ = ["JenkinsManager"]
38
39 logger = logging.getLogger(__name__)
40
41 _DEFAULT_TIMEOUT = object()
42
43
44 class JenkinsManager(object):
45 def __init__(self, jjb_config):
46 url = jjb_config.jenkins["url"]
47 user = jjb_config.jenkins["user"]
48 password = jjb_config.jenkins["password"]
49 timeout = jjb_config.jenkins["timeout"]
50
51 if timeout != _DEFAULT_TIMEOUT:
52 self.jenkins = jenkins.Jenkins(url, user, password, timeout)
53 else:
54 self.jenkins = jenkins.Jenkins(url, user, password)
55
56 self.cache = JobCache(
57 jjb_config.jenkins["url"], flush=jjb_config.builder["flush_cache"]
58 )
59
60 self._plugins_list = jjb_config.builder["plugins_info"]
61 self._jobs = None
62 self._job_list = None
63 self._views = None
64 self._view_list = None
65 self._jjb_config = jjb_config
66
67 def _setup_output(self, output, item, config_xml=False):
68 output_dir = output
69 output_fn = os.path.join(output, item)
70 if "/" in item:
71 # in item folder
72 output_fn = os.path.join(output, os.path.normpath(item))
73 output_dir = os.path.dirname(output_fn)
74
75 # if in a folder, re-adding name to the directory here
76 if config_xml:
77 output_dir = os.path.join(output_dir, os.path.basename(item))
78 output_fn = os.path.join(output_dir, "config.xml")
79
80 if output_dir != output:
81 logger.debug("Creating directory %s" % output_dir)
82 try:
83 os.makedirs(output_dir)
84 except OSError:
85 if not os.path.isdir(output_dir):
86 raise
87
88 return output_fn
89
90 @property
91 def jobs(self):
92 if self._jobs is None:
93 # populate jobs
94 self._jobs = self.jenkins.get_all_jobs()
95
96 return self._jobs
97
98 @property
99 def job_list(self):
100 if self._job_list is None:
101 # python-jenkins uses 'fullname' for folder/name combination
102 self._job_list = set(job["fullname"] for job in self.jobs)
103 return self._job_list
104
105 def _job_format(self, job_name):
106 # returns job name or url based on config option
107 if self._jjb_config.builder["print_job_urls"]:
108 return (
109 self._jjb_config.jenkins["url"]
110 + "/job/"
111 + quote("/job/".join(job_name.split("/")).encode("utf8"))
112 + "/"
113 )
114 else:
115 return job_name
116
117 def _view_format(self, view_name):
118 # returns job name or url based on config option
119 if self._jjb_config.builder["print_job_urls"]:
120 parts = view_name.split("/")
121 return (
122 self._jjb_config.jenkins["url"]
123 + "".join(["/job/" + item for item in parts[:-1]])
124 + "/view/"
125 + parts[-1]
126 + "/"
127 )
128 else:
129 return view_name
130
131 def update_job(self, job_name, xml):
132 if self.is_job(job_name):
133 logger.info(
134 "Reconfiguring jenkins job {0}".format(self._job_format(job_name))
135 )
136 self.jenkins.reconfig_job(job_name, xml)
137 else:
138 logger.info("Creating jenkins job {0}".format(self._job_format(job_name)))
139 self.jenkins.create_job(job_name, xml)
140
141 def is_job(self, job_name, use_cache=True):
142 if use_cache:
143 if job_name in self.job_list:
144 return True
145
146 return self.jenkins.job_exists(job_name)
147
148 def get_job_md5(self, job_name):
149 xml = self.jenkins.get_job_config(job_name)
150 return hashlib.md5(xml.encode("utf-8")).hexdigest()
151
152 def delete_job(self, job_name):
153 if self.is_job(job_name):
154 logger.info("Deleting jenkins job {0}".format(job_name))
155 self.jenkins.delete_job(job_name)
156
157 def get_plugins_info(self):
158 """Return a list of plugin_info dicts.
159
160 Returns one for each plugin on the Jenkins instance.
161 """
162 try:
163 plugins_list = self.jenkins.get_plugins().values()
164
165 except jenkins.JenkinsException as e:
166 if re.search("(Connection refused|Forbidden)", str(e)):
167 logger.warning(
168 "Unable to retrieve Jenkins Plugin Info from {0},"
169 " using default empty plugins info list.".format(
170 self.jenkins.server
171 )
172 )
173 plugins_list = [{"shortName": "", "version": "", "longName": ""}]
174 else:
175 raise
176 logger.debug("Jenkins Plugin Info {0}".format(pformat(plugins_list)))
177
178 return plugins_list
179
180 def get_jobs(self, cache=True):
181 if not cache:
182 self._jobs = None
183 self._job_list = None
184 return self.jobs
185
186 def is_managed_job(self, job_name):
187 xml = self.jenkins.get_job_config(job_name)
188 try:
189 out = XML.fromstring(xml.encode("utf-8"))
190 description = out.find(".//description").text
191 return description.endswith(MAGIC_MANAGE_STRING)
192 except (TypeError, AttributeError):
193 pass
194 return False
195
196 @property
197 def plugins_list(self):
198 if self._plugins_list is None:
199 self._plugins_list = self.get_plugins_info()
200 return self._plugins_list
201
202 def delete_old_managed_jobs(self, keep=None):
203 jobs = self.get_jobs()
204 deleted_jobs = 0
205 if keep is None:
206 keep = []
207 for job in jobs:
208 # python-jenkins stores the folder and name as 'fullname'
209 # Check if the job was deleted when his parent folder was deleted
210 if job["fullname"] not in keep and self.is_job(
211 job["fullname"], use_cache=False
212 ):
213 if self.is_managed_job(job["fullname"]):
214 logger.info(
215 "Removing obsolete jenkins job {0}".format(job["fullname"])
216 )
217 self.delete_job(job["fullname"])
218 deleted_jobs += 1
219 else:
220 logger.debug(
221 "Not deleting unmanaged jenkins job %s", job["fullname"]
222 )
223 else:
224 logger.debug("Keeping job %s", job["fullname"])
225 return deleted_jobs
226
227 def delete_jobs(self, jobs):
228 if jobs is not None:
229 logger.info("Removing jenkins job(s): %s" % ", ".join(jobs))
230 for job in jobs:
231 self.delete_job(job)
232 if self.cache.is_cached(job):
233 self.cache.set(job, "")
234 self.cache.save()
235
236 def delete_all_jobs(self):
237 jobs = self.get_jobs()
238 logger.info("Number of jobs to delete: %d", len(jobs))
239 script = (
240 "for(job in jenkins.model.Jenkins.theInstance.getAllItems())"
241 " { job.delete(); }"
242 )
243 self.jenkins.run_script(script)
244 # Need to clear the JJB cache after deletion
245 self.cache.clear()
246
247 def changed(self, job):
248 md5 = job.md5()
249
250 changed = self._jjb_config.builder["ignore_cache"] or self.cache.has_changed(
251 job.name, md5
252 )
253 if not changed:
254 logger.debug("'{0}' has not changed".format(job.name))
255 return changed
256
257 def exists(self, job):
258 exists = self.jenkins.job_exists(job.name)
259 if not exists:
260 logger.debug("'{0}' does not currently exist".format(job.name))
261 return exists
262
263 def update_jobs(
264 self,
265 xml_jobs,
266 output=None,
267 n_workers=None,
268 existing_only=None,
269 config_xml=False,
270 ):
271 orig = time.time()
272
273 logger.info("Number of jobs generated: %d", len(xml_jobs))
274 xml_jobs.sort(key=AlphanumSort)
275
276 if output and not hasattr(output, "write") and not os.path.isdir(output):
277 logger.debug("Creating directory %s" % output)
278 try:
279 os.makedirs(output)
280 except OSError:
281 if not os.path.isdir(output):
282 raise
283
284 if output:
285 # ensure only wrapped once
286 if hasattr(output, "write"):
287 output = utils.wrap_stream(output)
288
289 for job in xml_jobs:
290 if hasattr(output, "write"):
291 # `output` is a file-like object
292 logger.info("Job name: %s", job.name)
293 logger.debug("Writing XML to '{0}'".format(output))
294 try:
295 output.write(job.output())
296 except IOError as exc:
297 if exc.errno == errno.EPIPE:
298 # EPIPE could happen if piping output to something
299 # that doesn't read the whole input (e.g.: the UNIX
300 # `head` command)
301 return
302 raise
303 continue
304
305 output_fn = self._setup_output(output, job.name, config_xml)
306
307 logger.debug("Writing XML to '{0}'".format(output_fn))
308 with io.open(output_fn, "w", encoding="utf-8") as f:
309 f.write(job.output().decode("utf-8"))
310 return xml_jobs, len(xml_jobs)
311
312 # Filter out the jobs that did not change
313 logging.debug("Filtering %d jobs for changed jobs", len(xml_jobs))
314 step = time.time()
315 jobs = [job for job in xml_jobs if self.changed(job)]
316 logging.debug("Filtered for changed jobs in %ss", (time.time() - step))
317
318 if existing_only:
319 # Filter out the jobs not already in the cache
320 logging.debug("Filtering %d jobs for existing jobs", len(jobs))
321 step = time.time()
322 jobs = [job for job in jobs if self.exists(job)]
323 logging.debug("Filtered for existing jobs in %ss", (time.time() - step))
324
325 if not jobs:
326 return [], 0
327
328 # Update the jobs
329 logging.debug("Updating jobs")
330 step = time.time()
331 p_params = [{"job": job} for job in jobs]
332 results = self.parallel_update_job(n_workers=n_workers, concurrent=p_params)
333 logging.debug("Parsing results")
334 # generalize the result parsing, as a concurrent job always returns a
335 # list
336 if len(p_params) in (1, 0):
337 results = [results]
338 for result in results:
339 if isinstance(result, Exception):
340 raise result
341 else:
342 # update in-memory cache
343 j_name, j_md5 = result
344 self.cache.set(j_name, j_md5)
345 # write cache to disk
346 self.cache.save()
347 logging.debug("Updated %d jobs in %ss", len(jobs), time.time() - step)
348 logging.debug("Total run took %ss", (time.time() - orig))
349 return jobs, len(jobs)
350
351 @concurrent
352 def parallel_update_job(self, job):
353 self.fix_disabled(job)
354 self.update_job(job.name, job.output().decode("utf-8"))
355 return (job.name, job.md5())
356
357 def fix_disabled(self, job):
358 # allow disabled to remain the last setting
359 # see modules.general.General
360 el = job.xml.find("./disabled")
361 if el is not None:
362 return
363 if not self.is_job(job.name):
364 return
365 info = self.jenkins.get_job_info(job.name)
366 disabled = info.get("disabled", False)
367 XML.SubElement(job.xml, "disabled").text = str(disabled).lower()
368
369 ################
370 # View related #
371 ################
372
373 @property
374 def views(self):
375 if self._views is None:
376 # populate views
377 self._views = self.jenkins.get_views()
378 return self._views
379
380 @property
381 def view_list(self):
382 if self._view_list is None:
383 self._view_list = set(view["name"] for view in self.views)
384 return self._view_list
385
386 def get_views(self, cache=True):
387 if not cache:
388 self._views = None
389 self._view_list = None
390 return self.views
391
392 def is_view(self, view_name, use_cache=True):
393 if use_cache and view_name in self.view_list:
394 return True
395
396 # if not exists, use jenkins
397 return self.jenkins.view_exists(view_name)
398
399 def is_managed_view(self, view_name):
400 xml = self.jenkins.get_view_config(view_name)
401 try:
402 out = XML.fromstring(xml.encode("utf-8"))
403 description = out.find(".//description").text
404 return description.endswith(MAGIC_MANAGE_STRING)
405 except (TypeError, AttributeError):
406 pass
407 return False
408
409 def delete_old_managed_views(self, keep=None):
410 view_list = self.get_views()
411 deleted_views = 0
412 if keep is None:
413 keep = []
414 for view in view_list:
415 if view["name"] not in keep and self.is_view(view["name"], use_cache=False):
416 if self.is_managed_view(view["name"]):
417 logger.info(
418 "Removing obsolete jenkins view {0}".format(view["name"])
419 )
420 self.delete_view(view["name"])
421 deleted_views += 1
422 else:
423 logger.debug("Not deleting unmanaged jenkins view %s", view["name"])
424 else:
425 logger.debug("Keeping view %s", view["name"])
426 return deleted_views
427
428 def delete_view(self, view_name):
429 if self.is_view(view_name):
430 logger.info("Deleting jenkins view {}".format(view_name))
431 self.jenkins.delete_view(view_name)
432
433 def delete_views(self, views):
434 if views is not None:
435 logger.info("Removing jenkins view(s): %s" % ", ".join(views))
436 for view in views:
437 self.delete_view(view)
438 if self.cache.is_cached(view):
439 self.cache.set(view, "")
440 self.cache.save()
441
442 def delete_all_views(self):
443 views = self.get_views()
444 # Jenkins requires at least one view present. Don't remove the first
445 # view as it is likely the default view.
446 views.pop(0)
447 logger.info("Number of views to delete: %d", len(views))
448 for view in views:
449 self.delete_view(view["name"])
450 # Need to clear the JJB cache after deletion
451 self.cache.clear()
452
453 def update_view(self, view_name, xml):
454 if self.is_view(view_name):
455 logger.info(
456 "Reconfiguring jenkins view {0}".format(self._view_format(view_name))
457 )
458 self.jenkins.reconfig_view(view_name, xml)
459 else:
460 logger.info(
461 "Creating jenkins view {0}".format(self._view_format(view_name))
462 )
463 self.jenkins.create_view(view_name, xml)
464
465 def update_views(
466 self,
467 xml_views,
468 output=None,
469 n_workers=None,
470 existing_only=None,
471 config_xml=False,
472 ):
473 orig = time.time()
474
475 logger.info("Number of views generated: %d", len(xml_views))
476 xml_views.sort(key=AlphanumSort)
477
478 if output:
479 # ensure only wrapped once
480 if hasattr(output, "write"):
481 output = utils.wrap_stream(output)
482
483 for view in xml_views:
484 if hasattr(output, "write"):
485 # `output` is a file-like object
486 logger.info("View name: %s", view.name)
487 logger.debug("Writing XML to '{0}'".format(output))
488 try:
489 output.write(view.output())
490 except IOError as exc:
491 if exc.errno == errno.EPIPE:
492 # EPIPE could happen if piping output to something
493 # that doesn't read the whole input (e.g.: the UNIX
494 # `head` command)
495 return
496 raise
497 continue
498
499 output_fn = self._setup_output(output, view.name, config_xml)
500
501 logger.debug("Writing XML to '{0}'".format(output_fn))
502 with io.open(output_fn, "w", encoding="utf-8") as f:
503 f.write(view.output().decode("utf-8"))
504 return xml_views, len(xml_views)
505
506 # Filter out the views that did not change
507 logging.debug("Filtering %d views for changed views", len(xml_views))
508 step = time.time()
509 views = [view for view in xml_views if self.changed(view)]
510 logging.debug("Filtered for changed views in %ss", (time.time() - step))
511
512 if existing_only:
513 # Filter out the jobs not already in the cache
514 logging.debug("Filtering %d views for existing jobs", len(views))
515 step = time.time()
516 views = [view for view in views if self.exists(view)]
517 logging.debug("Filtered for existing views in %ss", (time.time() - step))
518
519 if not views:
520 return [], 0
521
522 # Update the views
523 logging.debug("Updating views")
524 step = time.time()
525 p_params = [{"view": view} for view in views]
526 results = self.parallel_update_view(n_workers=n_workers, concurrent=p_params)
527 logging.debug("Parsing results")
528 # generalize the result parsing, as a concurrent view always returns a
529 # list
530 if len(p_params) in (1, 0):
531 results = [results]
532 for result in results:
533 if isinstance(result, Exception):
534 raise result
535 else:
536 # update in-memory cache
537 v_name, v_md5 = result
538 self.cache.set(v_name, v_md5)
539 # write cache to disk
540 self.cache.save()
541 logging.debug("Updated %d views in %ss", len(views), time.time() - step)
542 logging.debug("Total run took %ss", (time.time() - orig))
543 return views, len(views)
544
545 @concurrent
546 def parallel_update_view(self, view):
547 self.update_view(view.name, view.output().decode("utf-8"))
548 return (view.name, view.md5())