| 1 |
#!/usr/bin/env python |
| 2 |
# Copyright (C) 2012 OpenStack, LLC. |
| 3 |
# |
| 4 |
# Licensed under the Apache License, Version 2.0 (the "License"); |
| 5 |
# you may not use this file except in compliance with the License. |
| 6 |
# You may obtain a copy of the License at |
| 7 |
# |
| 8 |
# http://www.apache.org/licenses/LICENSE-2.0 |
| 9 |
# |
| 10 |
# Unless required by applicable law or agreed to in writing, software |
| 11 |
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT |
| 12 |
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the |
| 13 |
# License for the specific language governing permissions and limitations |
| 14 |
# under the License. |
| 15 |
|
| 16 |
# Manage jobs in Jenkins server |
| 17 |
|
| 18 |
import errno |
| 19 |
import io |
| 20 |
import logging |
| 21 |
import os |
| 22 |
import re |
| 23 |
import tempfile |
| 24 |
|
| 25 |
import fasteners |
| 26 |
import yaml |
| 27 |
|
| 28 |
from jenkins_jobs import errors |
| 29 |
|
| 30 |
logger = logging.getLogger(__name__) |
| 31 |
|
| 32 |
|
| 33 |
class JobCache(object): |
| 34 |
# ensure each instance of the class has a reference to the required |
| 35 |
# modules so that they are available to be used when the destructor |
| 36 |
# is being called since python will not guarantee that it won't have |
| 37 |
# removed global module references during teardown. |
| 38 |
_logger = logger |
| 39 |
_os = os |
| 40 |
_tempfile = tempfile |
| 41 |
_yaml = yaml |
| 42 |
|
| 43 |
def __init__(self, jenkins_url, flush=False): |
| 44 |
cache_dir = self.get_cache_dir() |
| 45 |
# One cache per remote Jenkins URL: |
| 46 |
host_vary = re.sub(r"[^A-Za-z0-9\-\~]", "_", jenkins_url) |
| 47 |
self.cachefilename = os.path.join( |
| 48 |
cache_dir, "cache-host-jobs-" + host_vary + ".yml" |
| 49 |
) |
| 50 |
|
| 51 |
# generate named lockfile if none exists, and lock it |
| 52 |
self._locked = self._lock() |
| 53 |
if not self._locked: |
| 54 |
raise errors.JenkinsJobsException( |
| 55 |
"Unable to lock cache for '%s'" % jenkins_url |
| 56 |
) |
| 57 |
|
| 58 |
if flush or not os.path.isfile(self.cachefilename): |
| 59 |
self.data = {} |
| 60 |
else: |
| 61 |
with io.open(self.cachefilename, "r", encoding="utf-8") as yfile: |
| 62 |
self.data = yaml.safe_load(yfile) |
| 63 |
logger.debug("Using cache: '{0}'".format(self.cachefilename)) |
| 64 |
|
| 65 |
def _lock(self): |
| 66 |
self._fastener = fasteners.InterProcessLock("%s.lock" % self.cachefilename) |
| 67 |
|
| 68 |
return self._fastener.acquire(delay=1, max_delay=2, timeout=60) |
| 69 |
|
| 70 |
def _unlock(self): |
| 71 |
if getattr(self, "_locked", False): |
| 72 |
if getattr(self, "_fastener", None) is not None: |
| 73 |
self._fastener.release() |
| 74 |
self._locked = None |
| 75 |
|
| 76 |
@staticmethod |
| 77 |
def get_cache_dir(): |
| 78 |
home = os.path.expanduser("~") |
| 79 |
if home == "~": |
| 80 |
raise OSError("Could not locate home folder") |
| 81 |
xdg_cache_home = os.environ.get("XDG_CACHE_HOME") or os.path.join( |
| 82 |
home, ".cache" |
| 83 |
) |
| 84 |
path = os.path.join(xdg_cache_home, "jenkins_jobs") |
| 85 |
if not os.path.isdir(path): |
| 86 |
try: |
| 87 |
os.makedirs(path) |
| 88 |
except OSError as ose: |
| 89 |
# it could happen that two jjb instances are running at the |
| 90 |
# same time and that the other instance created the directory |
| 91 |
# after we made the check, in which case there is no error |
| 92 |
if ose.errno != errno.EEXIST: |
| 93 |
raise |
| 94 |
return path |
| 95 |
|
| 96 |
def set(self, job, md5): |
| 97 |
self.data[job] = md5 |
| 98 |
|
| 99 |
def clear(self): |
| 100 |
self.data.clear() |
| 101 |
|
| 102 |
def is_cached(self, job): |
| 103 |
if job in self.data: |
| 104 |
return True |
| 105 |
return False |
| 106 |
|
| 107 |
def has_changed(self, job, md5): |
| 108 |
if job in self.data and self.data[job] == md5: |
| 109 |
return False |
| 110 |
return True |
| 111 |
|
| 112 |
def save(self): |
| 113 |
# use self references to required modules in case called via __del__ |
| 114 |
# write to tempfile under same directory and then replace to avoid |
| 115 |
# issues around corruption such the process be killed |
| 116 |
tfile = self._tempfile.NamedTemporaryFile( |
| 117 |
dir=self.get_cache_dir(), delete=False |
| 118 |
) |
| 119 |
tfile.write(self._yaml.dump(self.data).encode("utf-8")) |
| 120 |
# force contents to be synced on disk before overwriting cachefile |
| 121 |
tfile.flush() |
| 122 |
self._os.fsync(tfile.fileno()) |
| 123 |
tfile.close() |
| 124 |
try: |
| 125 |
self._os.rename(tfile.name, self.cachefilename) |
| 126 |
except OSError: |
| 127 |
# On Windows, if dst already exists, OSError will be raised even if |
| 128 |
# it is a file. Remove the file first in that case and try again. |
| 129 |
self._os.remove(self.cachefilename) |
| 130 |
self._os.rename(tfile.name, self.cachefilename) |
| 131 |
|
| 132 |
self._logger.debug("Cache written out to '%s'" % self.cachefilename) |
| 133 |
|
| 134 |
def __del__(self): |
| 135 |
# check we initialized sufficiently in case called |
| 136 |
# due to an exception occurring in the __init__ |
| 137 |
if getattr(self, "data", None) is not None: |
| 138 |
try: |
| 139 |
self.save() |
| 140 |
except Exception as e: |
| 141 |
self._logger.error( |
| 142 |
"Failed to write to cache file '%s' on " |
| 143 |
"exit: %s" % (self.cachefilename, e) |
| 144 |
) |
| 145 |
self._unlock() |