Debian Codegraph @

jenkins_jobs/config.py

raw
1 #!/usr/bin/env python
2 # Copyright (C) 2015 Wayne Warren
3 #
4 # Licensed under the Apache License, Version 2.0 (the "License");
5 # you may not use this file except in compliance with the License.
6 # You may obtain a copy of the License at
7 #
8 # http://www.apache.org/licenses/LICENSE-2.0
9 #
10 # Unless required by applicable law or agreed to in writing, software
11 # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
12 # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
13 # License for the specific language governing permissions and limitations
14 # under the License.
15
16 # Manage JJB Configuration sources, defaults, and access.
17
18 from collections import defaultdict
19 import io
20 import logging
21 import os
22
23 from six.moves import configparser, StringIO
24 from six import PY2
25
26 from jenkins_jobs import builder
27 from jenkins_jobs.errors import JJBConfigException
28 from jenkins_jobs.errors import JenkinsJobsException
29
30 __all__ = ["JJBConfig"]
31
32 logger = logging.getLogger(__name__)
33
34 DEFAULT_CONF = """
35 [job_builder]
36 keep_descriptions=False
37 ignore_cache=False
38 recursive=False
39 exclude=.*
40 allow_duplicates=False
41 allow_empty_variables=False
42 retain_anchors=False
43 filter_modules=
44
45 # other named sections could be used in addition to the implicit [jenkins]
46 # if you have multiple jenkins servers.
47 [jenkins]
48 url=http://localhost:8080/
49 query_plugins_info=False
50 """
51
52 CONFIG_REQUIRED_MESSAGE = (
53 "A valid configuration file is required. " "No configuration file passed."
54 )
55 DEPRECATED_PLUGIN_CONFIG_SECTION_MESSAGE = (
56 "Defining plugin configuration using a [{plugin}] section in your config"
57 " file is deprecated. The recommended way to define plugins now is by"
58 ' using a [plugin "{plugin}"] section'
59 )
60 _NOTSET = object()
61
62
63 class JJBConfig(object):
64 def __init__(
65 self, config_filename=None, config_file_required=False, config_section="jenkins"
66 ):
67
68 """
69 The JJBConfig class is intended to encapsulate and resolve priority
70 between all sources of configuration for the JJB library. This allows
71 the various sources of configuration to provide a consistent accessor
72 interface regardless of where they are used.
73
74 It also allows users of JJB-as-an-API to create minimally valid
75 configuration and easily make minor modifications to default values
76 without strictly adhering to the confusing setup (see the _setup
77 method, the behavior of which largely lived in the cmd.execute method
78 previously) necessary for the jenkins-jobs command line tool.
79
80 :arg str config_filename: Name of configuration file on which to base
81 this config object.
82 :arg bool config_file_required: Allows users of the JJBConfig class to
83 decide whether or not it's really necessary for a config file to be
84 passed in when creating an instance. This has two effects on the
85 behavior of JJBConfig initialization:
86 * It determines whether or not we try "local" and "global" config
87 files.
88 * It determines whether or not failure to read some config file
89 will raise an exception or simply print a warning message
90 indicating that no config file was found.
91 """
92
93 config_parser = self._init_defaults()
94
95 global_conf = "/etc/jenkins_jobs/jenkins_jobs.ini"
96 user_conf = os.path.join(
97 os.path.expanduser("~"), ".config", "jenkins_jobs", "jenkins_jobs.ini"
98 )
99 local_conf = os.path.join(os.path.dirname(__file__), "jenkins_jobs.ini")
100 conf = None
101 if config_filename is not None:
102 conf = config_filename
103 else:
104 if os.path.isfile(local_conf):
105 conf = local_conf
106 elif os.path.isfile(user_conf):
107 conf = user_conf
108 else:
109 conf = global_conf
110
111 if config_file_required and conf is None:
112 raise JJBConfigException(CONFIG_REQUIRED_MESSAGE)
113
114 config_fp = None
115 if conf is not None:
116 try:
117 config_fp = self._read_config_file(conf)
118 except JJBConfigException:
119 if config_file_required:
120 raise JJBConfigException(CONFIG_REQUIRED_MESSAGE)
121 else:
122 logger.warning(
123 "Config file, {0}, not found. Using "
124 "default config values.".format(conf)
125 )
126
127 if config_fp is not None:
128 if PY2:
129 config_parser.readfp(config_fp)
130 else:
131 config_parser.read_file(config_fp)
132
133 self.config_parser = config_parser
134
135 self._section = config_section
136 self.print_job_urls = False
137
138 self.jenkins = defaultdict(None)
139 self.builder = defaultdict(None)
140 self.yamlparser = defaultdict(None)
141
142 self._setup()
143 self._handle_deprecated_hipchat_config()
144
145 if config_fp is not None:
146 config_fp.close()
147
148 def _init_defaults(self):
149 """Initialize default configuration values using DEFAULT_CONF."""
150 config = configparser.ConfigParser()
151 # Load default config always
152 if PY2:
153 config.readfp(StringIO(DEFAULT_CONF))
154 else:
155 config.read_file(StringIO(DEFAULT_CONF))
156 return config
157
158 def _read_config_file(self, config_filename):
159 """Return a ConfigParser object from a file input."""
160 if os.path.isfile(config_filename):
161 self.__config_file = config_filename # remember file we read from
162 logger.debug("Reading config from {0}".format(config_filename))
163 config_fp = io.open(config_filename, "r", encoding="utf-8")
164 else:
165 raise JJBConfigException(
166 "A valid configuration file is required. "
167 "\n{0} is not valid.".format(config_filename)
168 )
169
170 return config_fp
171
172 def _handle_deprecated_hipchat_config(self):
173 config = self.config_parser
174
175 if config.has_section("hipchat"):
176 if config.has_section('plugin "hipchat"'):
177 logger.warning(
178 'Both [hipchat] and [plugin "hipchat"] sections '
179 "defined, legacy [hipchat] section will be ignored."
180 )
181 else:
182 logger.warning(
183 "[hipchat] section is deprecated and should be moved to a "
184 '[plugins "hipchat"] section instead as the [hipchat] '
185 "section will be ignored in the future."
186 )
187 config.add_section('plugin "hipchat"')
188 for option in config.options("hipchat"):
189 config.set(
190 'plugin "hipchat"', option, config.get("hipchat", option)
191 )
192
193 config.remove_section("hipchat")
194
195 # remove need to reference jenkins section when using hipchat plugin
196 # moving to backports configparser would allow use of extended
197 # interpolation to remove the need for plugins to need information
198 # directly from the jenkins section within code and allow variables
199 # in the config file to refer instead.
200 if config.has_section('plugin "hipchat"') and not config.has_option(
201 'plugin "hipchat"', "url"
202 ):
203 config.set('plugin "hipchat"', "url", config.get("jenkins", "url"))
204
205 def _setup(self):
206 config = self.config_parser
207
208 logger.debug("Config: {0}".format(config))
209
210 # check the ignore_cache setting
211 ignore_cache = False
212 if config.has_option(self._section, "ignore_cache"):
213 logger.warning(
214 "ignore_cache option should be moved to the "
215 "[job_builder] section in the config file, the "
216 "one specified in the [jenkins] section will be "
217 "ignored in the future"
218 )
219 ignore_cache = config.getboolean(self._section, "ignore_cache")
220 elif config.has_option("job_builder", "ignore_cache"):
221 ignore_cache = config.getboolean("job_builder", "ignore_cache")
222 self.builder["ignore_cache"] = ignore_cache
223
224 # check the flush_cache setting
225 flush_cache = False
226 if config.has_option("job_builder", "flush_cache"):
227 flush_cache = config.getboolean("job_builder", "flush_cache")
228 self.builder["flush_cache"] = flush_cache
229
230 # check the print_job_urls setting
231 if config.has_option("job_builder", "print_job_urls"):
232 self.print_job_urls = config.getboolean("job_builder", "print_job_urls")
233
234 # Jenkins supports access as an anonymous user, which can be used to
235 # ensure read-only behaviour when querying the version of plugins
236 # installed for test mode to generate XML output matching what will be
237 # uploaded. To enable must pass 'None' as the value for user and
238 # password to python-jenkins
239 #
240 # catching 'TypeError' is a workaround for python 2.6 interpolation
241 # error
242 # https://bugs.launchpad.net/openstack-ci/+bug/1259631
243
244 try:
245 user = config.get(self._section, "user")
246 except (TypeError, configparser.NoOptionError):
247 user = None
248 self.jenkins["user"] = user
249
250 try:
251 password = config.get(self._section, "password")
252 except (TypeError, configparser.NoOptionError):
253 password = None
254 self.jenkins["password"] = password
255
256 # None -- no timeout, blocking mode; same as setblocking(True)
257 # 0.0 -- non-blocking mode; same as setblocking(False) <--- default
258 # > 0 -- timeout mode; operations time out after timeout seconds
259 # < 0 -- illegal; raises an exception
260 # to retain the default must use
261 # "timeout=jenkins_jobs.builder._DEFAULT_TIMEOUT" or not set timeout at
262 # all.
263 try:
264 timeout = config.getfloat(self._section, "timeout")
265 except (ValueError):
266 raise JenkinsJobsException("Jenkins timeout config is invalid")
267 except (TypeError, configparser.NoOptionError):
268 timeout = builder._DEFAULT_TIMEOUT
269 self.jenkins["timeout"] = timeout
270
271 plugins_info = None
272 if config.has_option(
273 self._section, "query_plugins_info"
274 ) and not config.getboolean(self._section, "query_plugins_info"):
275 logger.debug("Skipping plugin info retrieval")
276 plugins_info = []
277 self.builder["plugins_info"] = plugins_info
278
279 self.recursive = config.getboolean("job_builder", "recursive")
280 self.excludes = config.get("job_builder", "exclude").split(os.pathsep)
281
282 # The way we want to do things moving forward:
283 self.jenkins["url"] = config.get(self._section, "url")
284 self.builder["print_job_urls"] = self.print_job_urls
285
286 # keep descriptions ? (used by yamlparser)
287 keep_desc = False
288 if (
289 config
290 and config.has_section("job_builder")
291 and config.has_option("job_builder", "keep_descriptions")
292 ):
293 keep_desc = config.getboolean("job_builder", "keep_descriptions")
294 self.yamlparser["keep_descriptions"] = keep_desc
295
296 # figure out the include path (used by yamlparser)
297 path = ["."]
298 if (
299 config
300 and config.has_section("job_builder")
301 and config.has_option("job_builder", "include_path")
302 ):
303 path = config.get("job_builder", "include_path").split(":")
304 self.yamlparser["include_path"] = path
305
306 # Extra modules to load for jinja2 filters
307 filter_modules = []
308 if (
309 config
310 and config.has_section("job_builder")
311 and config.has_option("job_builder", "filter_modules")
312 and config.get("job_builder", "filter_modules")
313 ):
314 filter_modules = config.get("job_builder", "filter_modules").split(" ")
315 self.yamlparser["filter_modules"] = filter_modules
316
317 # allow duplicates?
318 allow_duplicates = False
319 if config and config.has_option("job_builder", "allow_duplicates"):
320 allow_duplicates = config.getboolean("job_builder", "allow_duplicates")
321 self.yamlparser["allow_duplicates"] = allow_duplicates
322
323 # allow empty variables?
324 self.yamlparser["allow_empty_variables"] = (
325 config
326 and config.has_section("job_builder")
327 and config.has_option("job_builder", "allow_empty_variables")
328 and config.getboolean("job_builder", "allow_empty_variables")
329 )
330
331 # retain anchors across files?
332 retain_anchors = False
333 if config and config.has_option("job_builder", "retain_anchors"):
334 retain_anchors = config.getboolean("job_builder", "retain_anchors")
335 self.yamlparser["retain_anchors"] = retain_anchors
336
337 update = None
338 if (
339 config
340 and config.has_section("job_builder")
341 and config.has_option("job_builder", "update")
342 ):
343 update = config.get("job_builder", "update")
344 self.builder["update"] = update
345
346 def validate(self):
347 # Inform the user as to what is likely to happen, as they may specify
348 # a real jenkins instance in test mode to get the plugin info to check
349 # the XML generated.
350 if self.jenkins["user"] is None and self.jenkins["password"] is None:
351 logger.info("Will use anonymous access to Jenkins if needed.")
352 elif (
353 self.jenkins["user"] is not None and self.jenkins["password"] is None
354 ) or (self.jenkins["user"] is None and self.jenkins["password"] is not None):
355 raise JenkinsJobsException(
356 "Cannot authenticate to Jenkins with only one of User and "
357 "Password provided, please check your configuration."
358 )
359
360 if self.builder["plugins_info"] is not None and not isinstance(
361 self.builder["plugins_info"], list
362 ):
363 raise JenkinsJobsException("plugins_info must contain a list!")
364
365 def get_module_config(self, section, key, default=None):
366 """Returns the value of a config in a config module.
367
368 Given a section name and a key value, return the value assigned to
369 the key in the JJB .ini file if it exists, otherwise emit a warning
370 indicating that the value is not set. Default value returned if no
371 value is set in the file will be a blank string.
372 """
373 result = default
374 try:
375 result = self.config_parser.get(section, key)
376 except (
377 configparser.NoSectionError,
378 configparser.NoOptionError,
379 JenkinsJobsException,
380 ) as e:
381 # use of default ignores missing sections/options
382 if result is None:
383 logger.warning(
384 "You didn't set a %s neither in the yaml job definition "
385 "nor in the %s section, blank default value will be "
386 "applied:\n%s",
387 key,
388 section,
389 e,
390 )
391 return result
392
393 def get_plugin_config(self, plugin, key, default=None):
394 return self.get_module_config('plugin "{}"'.format(plugin), key, default)