Debian Codegraph @

jenkins_jobs/registry.py

raw
1 #!/usr/bin/env python
2 # Copyright (C) 2015 OpenStack, LLC.
3 #
4 # Licensed under the Apache License, Version 2.0 (the "License");
5 # you may not use this file except in compliance with the License.
6 # You may obtain a copy of the License at
7 #
8 # http://www.apache.org/licenses/LICENSE-2.0
9 #
10 # Unless required by applicable law or agreed to in writing, software
11 # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
12 # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
13 # License for the specific language governing permissions and limitations
14 # under the License.
15
16 # Manage Jenkins plugin module registry.
17
18 import inspect
19 import logging
20 import operator
21 import packaging.version
22 import pkg_resources
23 import sys
24 import types
25
26 from six import PY2
27
28 from jenkins.plugins import PluginVersion
29 from jenkins_jobs.errors import JenkinsJobsException
30
31 __all__ = ["ModuleRegistry"]
32
33 logger = logging.getLogger(__name__)
34
35 getargspec = inspect.getargspec if PY2 else inspect.getfullargspec
36
37
38 class ModuleRegistry(object):
39 _entry_points_cache = {}
40 _component_type_cache = {}
41
42 def __init__(self, jjb_config, plugins_list=None):
43 self.modules = []
44 self.modules_by_component_type = {}
45 self.handlers = {}
46 self.jjb_config = jjb_config
47 self.masked_warned = {}
48 self._macros = {}
49
50 if plugins_list is None:
51 self._plugin_version = {}
52 else:
53 # PluginVersion by short and long plugin name.
54 self._plugin_version = self._get_plugins_versions(plugins_list)
55
56 for entrypoint in pkg_resources.iter_entry_points(group="jenkins_jobs.modules"):
57 Mod = entrypoint.load()
58 mod = Mod(self)
59 self.modules.append(mod)
60 self.modules.sort(key=operator.attrgetter("sequence"))
61 if mod.component_type is not None:
62 self.modules_by_component_type[mod.component_type] = entrypoint
63
64 @staticmethod
65 def _get_plugins_versions(plugins_list):
66 plugin_version = {}
67
68 for plugin_info in plugins_list:
69 short_name = plugin_info.get("shortName")
70 long_name = plugin_info.get("longName")
71
72 version = plugin_info["version"]
73 if version == None: # noqa: E711; should call PluginInfo.__eq__.
74 # Ensure that plugin_info always has version and it is instance of PluginVersion.
75 plugin_info["version"] = str(sys.maxsize)
76 version = plugin_info["version"]
77
78 try:
79 packaging.version.Version(version)
80 except packaging.version.InvalidVersion:
81 plugin_name = short_name or long_name
82 if plugin_name:
83 logger.warning(
84 "Version %s for plugin %s does not conform to PEP440",
85 version,
86 plugin_name,
87 )
88 else:
89 logger.warning("Version %s does not conform to PEP440", version)
90
91 if short_name:
92 plugin_version[short_name] = version
93 if long_name:
94 plugin_version[long_name] = version
95
96 return plugin_version
97
98 @staticmethod
99 def _filter_kwargs(func, **kwargs):
100 arg_spec = getargspec(func)
101 for name in list(kwargs.keys()):
102 if name not in arg_spec.args:
103 del kwargs[name]
104 return kwargs
105
106 def get_plugin_version(self, plugin_name, alt_plugin_name=None, default=None):
107 """Provide plugin version to be used from a module's impl of Base.gen_xml.
108
109 The return value is a plugin version obtained directly from a running
110 Jenkins instance.
111 This allows module authors to differentiate generated XML output based
112 on it.
113
114 :arg str plugin_name: Either the shortName or longName of a plugin
115 as seen in a query that looks like:
116 ``http://<jenkins-hostname>/pluginManager/api/json?pretty&depth=2``
117 :arg str alt_plugin_name: Alternative plugin name. Used if plugin_name
118 is missing in plugin list.
119 :arg str default: Default value. Used if plugin name is missing in
120 plugin list.
121
122 During a 'test' run, it is possible to override JJB's query to a live
123 Jenkins instance by passing it a path to a file containing a YAML list
124 of dictionaries that mimics the plugin properties you want your test
125 output to reflect::
126
127 jenkins-jobs test -p /path/to/plugins-info.yaml
128
129 Below is example YAML that might be included in
130 /path/to/plugins-info.yaml.
131
132 .. literalinclude:: /../../tests/cmd/fixtures/plugins-info.yaml
133
134 """
135 try:
136 return self._plugin_version[plugin_name]
137 except KeyError:
138 pass
139 if alt_plugin_name:
140 try:
141 return self._plugin_version[alt_plugin_name]
142 except KeyError:
143 pass
144 if default is not None:
145 return PluginVersion(default)
146 # Assume latest version of plugin is preferred config format.
147 return PluginVersion(str(sys.maxsize))
148
149 def registerHandler(self, category, name, method):
150 cat_dict = self.handlers.get(category, {})
151 if not cat_dict:
152 self.handlers[category] = cat_dict
153 cat_dict[name] = method
154
155 def getHandler(self, category, name):
156 return self.handlers[category][name]
157
158 @property
159 def macros(self):
160 return self._macros
161
162 def set_macros(self, macros):
163 self._macros = macros
164
165 def amend_job_dicts(self, job_data_list):
166 while True:
167 changed = False
168 for job in job_data_list:
169 for module in self.modules:
170 if module.amend_job_dict(job.data):
171 changed = True
172 if not changed:
173 break
174
175 def get_component_list_type(self, entry_point):
176 if entry_point in self._component_type_cache:
177 return self._component_type_cache[entry_point]
178
179 # pkg_resources.EntryPoint.load() is costly, cache it.
180 component_list_type = entry_point.load().component_list_type
181 logging.info("Caching type %s of %s", component_list_type, entry_point)
182 self._component_type_cache[entry_point] = component_list_type
183
184 return component_list_type
185
186 def dispatch(
187 self,
188 component_type,
189 xml_parent,
190 component,
191 template_data={},
192 job_data=None,
193 component_pos=None,
194 ):
195 """This is a method that you can call from your implementation of
196 Base.gen_xml or component. It allows modules to define a type
197 of component, and benefit from extensibility via Python
198 entry points and Jenkins Job Builder :ref:`Macros <macro>`.
199
200 :arg str component_type: the name of the component
201 (e.g., `builder`)
202 :arg xml_parent: the parent XML element
203 :arg component: component definition
204 :arg dict template_data: values that should be interpolated into
205 the component definition
206 :arg dict job_data: full job definition
207
208 See :py:class:`jenkins_jobs.modules.base.Base` for how to register
209 components of a module.
210
211 See the Publishers module for a simple example of how to use
212 this method.
213 """
214
215 if component_type not in self.modules_by_component_type:
216 raise JenkinsJobsException(
217 "Unknown component type: " "'{0}'.".format(component_type)
218 )
219
220 entry_point = self.modules_by_component_type[component_type]
221 component_list_type = self.get_component_list_type(entry_point)
222
223 if isinstance(component, dict):
224 # The component is a singleton dictionary of name: dict(args)
225 name, component_data = next(iter(component.items()))
226 if template_data:
227 paramdict = {}
228 paramdict.update(template_data)
229 paramdict.update(job_data or {})
230 else:
231 # The component is a simple string name, eg "run-tests"
232 name = component
233 component_data = {}
234
235 # Look for a component function defined in an entry point
236 eps = self._entry_points_cache.get(component_list_type)
237 if eps is None:
238 eps = self._load_eps(component_list_type, component_type, entry_point, name)
239
240 macro_dict = self.macros.get(component_type, {})
241 macro = macro_dict.get(name)
242 if macro:
243 try:
244 self._dispatch_macro(
245 component_data,
246 component_type,
247 eps,
248 job_data,
249 macro,
250 name,
251 xml_parent,
252 )
253 except JenkinsJobsException as x:
254 if component_pos is not None:
255 raise x.with_context(
256 f"While expanding {component_type} macro call {name!r}",
257 pos=component_pos,
258 )
259 else:
260 raise
261 elif name in eps:
262 try:
263 func = eps[name]
264 kwargs = self._filter_kwargs(func, job_data=job_data)
265 func(self, xml_parent, component_data, **kwargs)
266 except JenkinsJobsException as x:
267 raise x.with_context(
268 f"In {component_type} {name!r}",
269 pos=component.pos,
270 )
271 else:
272 raise JenkinsJobsException(
273 "Unknown entry point or macro '{0}' "
274 "for component type: '{1}'.".format(name, component_type)
275 )
276
277 def _dispatch_macro(
278 self, component_data, component_type, eps, job_data, macro, name, xml_parent
279 ):
280 if name in eps and name not in self.masked_warned:
281 self.masked_warned[name] = True
282 logger.warning(
283 "You have a macro ('%s') defined for '%s' "
284 "component type that is masking an inbuilt "
285 "definition" % (name, component_type)
286 )
287 if component_data is None:
288 component_data = {}
289 params = {**component_data, **(job_data or {})}
290 macro.dispatch_elements(self, xml_parent, component_data, job_data, params)
291
292 def _load_eps(self, component_list_type, component_type, entry_point, name):
293 logging.debug("Caching entrypoints for %s" % component_list_type)
294 module_eps = []
295 # auto build entry points by inferring from base component_types
296 mod = pkg_resources.EntryPoint(
297 "__all__", entry_point.module_name, dist=entry_point.dist
298 )
299 Mod = mod.load()
300 func_eps = [
301 Mod.__dict__.get(a)
302 for a in dir(Mod)
303 if isinstance(Mod.__dict__.get(a), types.FunctionType)
304 ]
305 for func_ep in func_eps:
306 try:
307 # extract entry point based on docstring
308 name_line = func_ep.__doc__.split("\n")
309 if not name_line[0].startswith("yaml:"):
310 logger.debug("Ignoring '%s' as an entry point" % name_line)
311 continue
312 ep_name = name_line[0].split(" ")[1]
313 except (AttributeError, IndexError):
314 # AttributeError by docstring not being defined as
315 # a string to have split called on it.
316 # IndexError raised by name_line not containing anything
317 # after the 'yaml:' string.
318 logger.debug(
319 "Not including func '%s' as an entry point" % func_ep.__name__
320 )
321 continue
322
323 module_eps.append(
324 pkg_resources.EntryPoint(
325 ep_name,
326 entry_point.module_name,
327 dist=entry_point.dist,
328 attrs=(func_ep.__name__,),
329 )
330 )
331 logger.debug(
332 "Adding auto EP '%s=%s:%s'"
333 % (ep_name, entry_point.module_name, func_ep.__name__)
334 )
335 # load from explicitly defined entry points
336 module_eps.extend(
337 list(
338 pkg_resources.iter_entry_points(
339 group="jenkins_jobs.{0}".format(component_list_type)
340 )
341 )
342 )
343 eps = {}
344 for module_ep in module_eps:
345 if module_ep.name in eps:
346 raise JenkinsJobsException(
347 "Duplicate entry point found for component type: "
348 "'{0}', '{0}',"
349 "name: '{1}'".format(component_type, name)
350 )
351
352 eps[module_ep.name] = module_ep.load()
353 # cache both sets of entry points
354 self._entry_points_cache[component_list_type] = eps
355 logger.debug("Cached entry point group %s = %s", component_list_type, eps)
356 return eps