| 1 |
#!/usr/bin/env python |
| 2 |
# Copyright (C) 2015 OpenStack, LLC. |
| 3 |
# |
| 4 |
# Licensed under the Apache License, Version 2.0 (the "License"); |
| 5 |
# you may not use this file except in compliance with the License. |
| 6 |
# You may obtain a copy of the License at |
| 7 |
# |
| 8 |
# http://www.apache.org/licenses/LICENSE-2.0 |
| 9 |
# |
| 10 |
# Unless required by applicable law or agreed to in writing, software |
| 11 |
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT |
| 12 |
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the |
| 13 |
# License for the specific language governing permissions and limitations |
| 14 |
# under the License. |
| 15 |
|
| 16 |
# Manage Jenkins plugin module registry. |
| 17 |
|
| 18 |
import inspect |
| 19 |
import logging |
| 20 |
import operator |
| 21 |
import packaging.version |
| 22 |
import pkg_resources |
| 23 |
import sys |
| 24 |
import types |
| 25 |
|
| 26 |
from six import PY2 |
| 27 |
|
| 28 |
from jenkins.plugins import PluginVersion |
| 29 |
from jenkins_jobs.errors import JenkinsJobsException |
| 30 |
|
| 31 |
__all__ = ["ModuleRegistry"] |
| 32 |
|
| 33 |
logger = logging.getLogger(__name__) |
| 34 |
|
| 35 |
getargspec = inspect.getargspec if PY2 else inspect.getfullargspec |
| 36 |
|
| 37 |
|
| 38 |
class ModuleRegistry(object): |
| 39 |
_entry_points_cache = {} |
| 40 |
_component_type_cache = {} |
| 41 |
|
| 42 |
def __init__(self, jjb_config, plugins_list=None): |
| 43 |
self.modules = [] |
| 44 |
self.modules_by_component_type = {} |
| 45 |
self.handlers = {} |
| 46 |
self.jjb_config = jjb_config |
| 47 |
self.masked_warned = {} |
| 48 |
self._macros = {} |
| 49 |
|
| 50 |
if plugins_list is None: |
| 51 |
self._plugin_version = {} |
| 52 |
else: |
| 53 |
# PluginVersion by short and long plugin name. |
| 54 |
self._plugin_version = self._get_plugins_versions(plugins_list) |
| 55 |
|
| 56 |
for entrypoint in pkg_resources.iter_entry_points(group="jenkins_jobs.modules"): |
| 57 |
Mod = entrypoint.load() |
| 58 |
mod = Mod(self) |
| 59 |
self.modules.append(mod) |
| 60 |
self.modules.sort(key=operator.attrgetter("sequence")) |
| 61 |
if mod.component_type is not None: |
| 62 |
self.modules_by_component_type[mod.component_type] = entrypoint |
| 63 |
|
| 64 |
@staticmethod |
| 65 |
def _get_plugins_versions(plugins_list): |
| 66 |
plugin_version = {} |
| 67 |
|
| 68 |
for plugin_info in plugins_list: |
| 69 |
short_name = plugin_info.get("shortName") |
| 70 |
long_name = plugin_info.get("longName") |
| 71 |
|
| 72 |
version = plugin_info["version"] |
| 73 |
if version == None: # noqa: E711; should call PluginInfo.__eq__. |
| 74 |
# Ensure that plugin_info always has version and it is instance of PluginVersion. |
| 75 |
plugin_info["version"] = str(sys.maxsize) |
| 76 |
version = plugin_info["version"] |
| 77 |
|
| 78 |
try: |
| 79 |
packaging.version.Version(version) |
| 80 |
except packaging.version.InvalidVersion: |
| 81 |
plugin_name = short_name or long_name |
| 82 |
if plugin_name: |
| 83 |
logger.warning( |
| 84 |
"Version %s for plugin %s does not conform to PEP440", |
| 85 |
version, |
| 86 |
plugin_name, |
| 87 |
) |
| 88 |
else: |
| 89 |
logger.warning("Version %s does not conform to PEP440", version) |
| 90 |
|
| 91 |
if short_name: |
| 92 |
plugin_version[short_name] = version |
| 93 |
if long_name: |
| 94 |
plugin_version[long_name] = version |
| 95 |
|
| 96 |
return plugin_version |
| 97 |
|
| 98 |
@staticmethod |
| 99 |
def _filter_kwargs(func, **kwargs): |
| 100 |
arg_spec = getargspec(func) |
| 101 |
for name in list(kwargs.keys()): |
| 102 |
if name not in arg_spec.args: |
| 103 |
del kwargs[name] |
| 104 |
return kwargs |
| 105 |
|
| 106 |
def get_plugin_version(self, plugin_name, alt_plugin_name=None, default=None): |
| 107 |
"""Provide plugin version to be used from a module's impl of Base.gen_xml. |
| 108 |
|
| 109 |
The return value is a plugin version obtained directly from a running |
| 110 |
Jenkins instance. |
| 111 |
This allows module authors to differentiate generated XML output based |
| 112 |
on it. |
| 113 |
|
| 114 |
:arg str plugin_name: Either the shortName or longName of a plugin |
| 115 |
as seen in a query that looks like: |
| 116 |
``http://<jenkins-hostname>/pluginManager/api/json?pretty&depth=2`` |
| 117 |
:arg str alt_plugin_name: Alternative plugin name. Used if plugin_name |
| 118 |
is missing in plugin list. |
| 119 |
:arg str default: Default value. Used if plugin name is missing in |
| 120 |
plugin list. |
| 121 |
|
| 122 |
During a 'test' run, it is possible to override JJB's query to a live |
| 123 |
Jenkins instance by passing it a path to a file containing a YAML list |
| 124 |
of dictionaries that mimics the plugin properties you want your test |
| 125 |
output to reflect:: |
| 126 |
|
| 127 |
jenkins-jobs test -p /path/to/plugins-info.yaml |
| 128 |
|
| 129 |
Below is example YAML that might be included in |
| 130 |
/path/to/plugins-info.yaml. |
| 131 |
|
| 132 |
.. literalinclude:: /../../tests/cmd/fixtures/plugins-info.yaml |
| 133 |
|
| 134 |
""" |
| 135 |
try: |
| 136 |
return self._plugin_version[plugin_name] |
| 137 |
except KeyError: |
| 138 |
pass |
| 139 |
if alt_plugin_name: |
| 140 |
try: |
| 141 |
return self._plugin_version[alt_plugin_name] |
| 142 |
except KeyError: |
| 143 |
pass |
| 144 |
if default is not None: |
| 145 |
return PluginVersion(default) |
| 146 |
# Assume latest version of plugin is preferred config format. |
| 147 |
return PluginVersion(str(sys.maxsize)) |
| 148 |
|
| 149 |
def registerHandler(self, category, name, method): |
| 150 |
cat_dict = self.handlers.get(category, {}) |
| 151 |
if not cat_dict: |
| 152 |
self.handlers[category] = cat_dict |
| 153 |
cat_dict[name] = method |
| 154 |
|
| 155 |
def getHandler(self, category, name): |
| 156 |
return self.handlers[category][name] |
| 157 |
|
| 158 |
@property |
| 159 |
def macros(self): |
| 160 |
return self._macros |
| 161 |
|
| 162 |
def set_macros(self, macros): |
| 163 |
self._macros = macros |
| 164 |
|
| 165 |
def amend_job_dicts(self, job_data_list): |
| 166 |
while True: |
| 167 |
changed = False |
| 168 |
for job in job_data_list: |
| 169 |
for module in self.modules: |
| 170 |
if module.amend_job_dict(job.data): |
| 171 |
changed = True |
| 172 |
if not changed: |
| 173 |
break |
| 174 |
|
| 175 |
def get_component_list_type(self, entry_point): |
| 176 |
if entry_point in self._component_type_cache: |
| 177 |
return self._component_type_cache[entry_point] |
| 178 |
|
| 179 |
# pkg_resources.EntryPoint.load() is costly, cache it. |
| 180 |
component_list_type = entry_point.load().component_list_type |
| 181 |
logging.info("Caching type %s of %s", component_list_type, entry_point) |
| 182 |
self._component_type_cache[entry_point] = component_list_type |
| 183 |
|
| 184 |
return component_list_type |
| 185 |
|
| 186 |
def dispatch( |
| 187 |
self, |
| 188 |
component_type, |
| 189 |
xml_parent, |
| 190 |
component, |
| 191 |
template_data={}, |
| 192 |
job_data=None, |
| 193 |
component_pos=None, |
| 194 |
): |
| 195 |
"""This is a method that you can call from your implementation of |
| 196 |
Base.gen_xml or component. It allows modules to define a type |
| 197 |
of component, and benefit from extensibility via Python |
| 198 |
entry points and Jenkins Job Builder :ref:`Macros <macro>`. |
| 199 |
|
| 200 |
:arg str component_type: the name of the component |
| 201 |
(e.g., `builder`) |
| 202 |
:arg xml_parent: the parent XML element |
| 203 |
:arg component: component definition |
| 204 |
:arg dict template_data: values that should be interpolated into |
| 205 |
the component definition |
| 206 |
:arg dict job_data: full job definition |
| 207 |
|
| 208 |
See :py:class:`jenkins_jobs.modules.base.Base` for how to register |
| 209 |
components of a module. |
| 210 |
|
| 211 |
See the Publishers module for a simple example of how to use |
| 212 |
this method. |
| 213 |
""" |
| 214 |
|
| 215 |
if component_type not in self.modules_by_component_type: |
| 216 |
raise JenkinsJobsException( |
| 217 |
"Unknown component type: " "'{0}'.".format(component_type) |
| 218 |
) |
| 219 |
|
| 220 |
entry_point = self.modules_by_component_type[component_type] |
| 221 |
component_list_type = self.get_component_list_type(entry_point) |
| 222 |
|
| 223 |
if isinstance(component, dict): |
| 224 |
# The component is a singleton dictionary of name: dict(args) |
| 225 |
name, component_data = next(iter(component.items())) |
| 226 |
if template_data: |
| 227 |
paramdict = {} |
| 228 |
paramdict.update(template_data) |
| 229 |
paramdict.update(job_data or {}) |
| 230 |
else: |
| 231 |
# The component is a simple string name, eg "run-tests" |
| 232 |
name = component |
| 233 |
component_data = {} |
| 234 |
|
| 235 |
# Look for a component function defined in an entry point |
| 236 |
eps = self._entry_points_cache.get(component_list_type) |
| 237 |
if eps is None: |
| 238 |
eps = self._load_eps(component_list_type, component_type, entry_point, name) |
| 239 |
|
| 240 |
macro_dict = self.macros.get(component_type, {}) |
| 241 |
macro = macro_dict.get(name) |
| 242 |
if macro: |
| 243 |
try: |
| 244 |
self._dispatch_macro( |
| 245 |
component_data, |
| 246 |
component_type, |
| 247 |
eps, |
| 248 |
job_data, |
| 249 |
macro, |
| 250 |
name, |
| 251 |
xml_parent, |
| 252 |
) |
| 253 |
except JenkinsJobsException as x: |
| 254 |
if component_pos is not None: |
| 255 |
raise x.with_context( |
| 256 |
f"While expanding {component_type} macro call {name!r}", |
| 257 |
pos=component_pos, |
| 258 |
) |
| 259 |
else: |
| 260 |
raise |
| 261 |
elif name in eps: |
| 262 |
try: |
| 263 |
func = eps[name] |
| 264 |
kwargs = self._filter_kwargs(func, job_data=job_data) |
| 265 |
func(self, xml_parent, component_data, **kwargs) |
| 266 |
except JenkinsJobsException as x: |
| 267 |
raise x.with_context( |
| 268 |
f"In {component_type} {name!r}", |
| 269 |
pos=component.pos, |
| 270 |
) |
| 271 |
else: |
| 272 |
raise JenkinsJobsException( |
| 273 |
"Unknown entry point or macro '{0}' " |
| 274 |
"for component type: '{1}'.".format(name, component_type) |
| 275 |
) |
| 276 |
|
| 277 |
def _dispatch_macro( |
| 278 |
self, component_data, component_type, eps, job_data, macro, name, xml_parent |
| 279 |
): |
| 280 |
if name in eps and name not in self.masked_warned: |
| 281 |
self.masked_warned[name] = True |
| 282 |
logger.warning( |
| 283 |
"You have a macro ('%s') defined for '%s' " |
| 284 |
"component type that is masking an inbuilt " |
| 285 |
"definition" % (name, component_type) |
| 286 |
) |
| 287 |
if component_data is None: |
| 288 |
component_data = {} |
| 289 |
params = {**component_data, **(job_data or {})} |
| 290 |
macro.dispatch_elements(self, xml_parent, component_data, job_data, params) |
| 291 |
|
| 292 |
def _load_eps(self, component_list_type, component_type, entry_point, name): |
| 293 |
logging.debug("Caching entrypoints for %s" % component_list_type) |
| 294 |
module_eps = [] |
| 295 |
# auto build entry points by inferring from base component_types |
| 296 |
mod = pkg_resources.EntryPoint( |
| 297 |
"__all__", entry_point.module_name, dist=entry_point.dist |
| 298 |
) |
| 299 |
Mod = mod.load() |
| 300 |
func_eps = [ |
| 301 |
Mod.__dict__.get(a) |
| 302 |
for a in dir(Mod) |
| 303 |
if isinstance(Mod.__dict__.get(a), types.FunctionType) |
| 304 |
] |
| 305 |
for func_ep in func_eps: |
| 306 |
try: |
| 307 |
# extract entry point based on docstring |
| 308 |
name_line = func_ep.__doc__.split("\n") |
| 309 |
if not name_line[0].startswith("yaml:"): |
| 310 |
logger.debug("Ignoring '%s' as an entry point" % name_line) |
| 311 |
continue |
| 312 |
ep_name = name_line[0].split(" ")[1] |
| 313 |
except (AttributeError, IndexError): |
| 314 |
# AttributeError by docstring not being defined as |
| 315 |
# a string to have split called on it. |
| 316 |
# IndexError raised by name_line not containing anything |
| 317 |
# after the 'yaml:' string. |
| 318 |
logger.debug( |
| 319 |
"Not including func '%s' as an entry point" % func_ep.__name__ |
| 320 |
) |
| 321 |
continue |
| 322 |
|
| 323 |
module_eps.append( |
| 324 |
pkg_resources.EntryPoint( |
| 325 |
ep_name, |
| 326 |
entry_point.module_name, |
| 327 |
dist=entry_point.dist, |
| 328 |
attrs=(func_ep.__name__,), |
| 329 |
) |
| 330 |
) |
| 331 |
logger.debug( |
| 332 |
"Adding auto EP '%s=%s:%s'" |
| 333 |
% (ep_name, entry_point.module_name, func_ep.__name__) |
| 334 |
) |
| 335 |
# load from explicitly defined entry points |
| 336 |
module_eps.extend( |
| 337 |
list( |
| 338 |
pkg_resources.iter_entry_points( |
| 339 |
group="jenkins_jobs.{0}".format(component_list_type) |
| 340 |
) |
| 341 |
) |
| 342 |
) |
| 343 |
eps = {} |
| 344 |
for module_ep in module_eps: |
| 345 |
if module_ep.name in eps: |
| 346 |
raise JenkinsJobsException( |
| 347 |
"Duplicate entry point found for component type: " |
| 348 |
"'{0}', '{0}'," |
| 349 |
"name: '{1}'".format(component_type, name) |
| 350 |
) |
| 351 |
|
| 352 |
eps[module_ep.name] = module_ep.load() |
| 353 |
# cache both sets of entry points |
| 354 |
self._entry_points_cache[component_list_type] = eps |
| 355 |
logger.debug("Cached entry point group %s = %s", component_list_type, eps) |
| 356 |
return eps |