| 1 |
# Licensed under the Apache License, Version 2.0 (the "License"); |
| 2 |
# you may not use this file except in compliance with the License. |
| 3 |
# You may obtain a copy of the License at |
| 4 |
# |
| 5 |
# http://www.apache.org/licenses/LICENSE-2.0 |
| 6 |
# |
| 7 |
# Unless required by applicable law or agreed to in writing, software |
| 8 |
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT |
| 9 |
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the |
| 10 |
# License for the specific language governing permissions and limitations |
| 11 |
# under the License. |
| 12 |
|
| 13 |
from functools import partial |
| 14 |
from itertools import filterfalse |
| 15 |
from jinja2 import StrictUndefined |
| 16 |
|
| 17 |
from .errors import Context, JenkinsJobsException |
| 18 |
from .formatter import CustomFormatter, enum_str_format_required_params |
| 19 |
from .loc_loader import LocDict, LocString, LocList |
| 20 |
from .yaml_objects import ( |
| 21 |
J2String, |
| 22 |
J2Yaml, |
| 23 |
YamlInclude, |
| 24 |
YamlListJoin, |
| 25 |
IncludeJinja2, |
| 26 |
IncludeRawExpand, |
| 27 |
IncludeRawVerbatim, |
| 28 |
) |
| 29 |
|
| 30 |
|
| 31 |
def expand_dict(expander, obj, params, key_pos, value_pos): |
| 32 |
result = LocDict(pos=obj.pos) |
| 33 |
for key, value in obj.items(): |
| 34 |
expanded_key = expander.expand(key, params, None) |
| 35 |
expanded_value = expander.expand( |
| 36 |
value, params, obj.key_pos.get(key), obj.value_pos.get(key) |
| 37 |
) |
| 38 |
result.set_item( |
| 39 |
expanded_key, expanded_value, obj.key_pos.get(key), obj.value_pos.get(key) |
| 40 |
) |
| 41 |
return result |
| 42 |
|
| 43 |
|
| 44 |
def expand_list(expander, obj, params, key_pos, value_pos): |
| 45 |
items = [ |
| 46 |
expander.expand(item, params, None, obj.value_pos[idx]) |
| 47 |
for idx, item in enumerate(obj) |
| 48 |
] |
| 49 |
value_pos = [obj.value_pos[idx] for idx, _ in enumerate(obj)] |
| 50 |
return LocList(items, obj.pos, value_pos) |
| 51 |
|
| 52 |
|
| 53 |
def expand_tuple(expander, obj, params, key_pos, value_pos): |
| 54 |
return tuple(expander.expand(item, params, None) for item in obj) |
| 55 |
|
| 56 |
|
| 57 |
class StrExpander: |
| 58 |
def __init__(self, allow_empty_variables): |
| 59 |
self._formatter = CustomFormatter(allow_empty_variables) |
| 60 |
|
| 61 |
def __call__(self, obj, params, key_pos, value_pos): |
| 62 |
try: |
| 63 |
return self._formatter.format(str(obj), **params) |
| 64 |
except JenkinsJobsException as x: |
| 65 |
lines = str(obj).splitlines() |
| 66 |
start_ofs = value_pos.body.index(lines[0]) |
| 67 |
pre_pad = value_pos.body[:start_ofs] |
| 68 |
# Shift position to reflect template position inside yaml file: |
| 69 |
if "\n" in pre_pad: |
| 70 |
pos = value_pos.with_offset(line_ofs=1) |
| 71 |
else: |
| 72 |
pos = value_pos.with_offset(column_ofs=start_ofs) |
| 73 |
pos = pos.with_contents_start() |
| 74 |
raise x.with_pos(pos) |
| 75 |
|
| 76 |
|
| 77 |
def call_expand(expander, obj, params, key_pos, value_pos): |
| 78 |
return obj.expand(expander, params) |
| 79 |
|
| 80 |
|
| 81 |
def dont_expand(obj, params, key_pos, value_pos): |
| 82 |
return obj |
| 83 |
|
| 84 |
|
| 85 |
def dont_expand_yaml_object(expander, obj, params, key_pos, value_pos): |
| 86 |
return obj |
| 87 |
|
| 88 |
|
| 89 |
yaml_classes_list = [ |
| 90 |
J2String, |
| 91 |
J2Yaml, |
| 92 |
YamlInclude, |
| 93 |
YamlListJoin, |
| 94 |
IncludeJinja2, |
| 95 |
IncludeRawExpand, |
| 96 |
IncludeRawVerbatim, |
| 97 |
] |
| 98 |
|
| 99 |
deprecated_yaml_tags = [ |
| 100 |
("!include", YamlInclude), |
| 101 |
("!include-raw", IncludeRawExpand), |
| 102 |
("!include-raw:", IncludeRawExpand), |
| 103 |
("!include-raw-escape", IncludeRawVerbatim), |
| 104 |
("!include-raw-escape:", IncludeRawVerbatim), |
| 105 |
] |
| 106 |
|
| 107 |
|
| 108 |
# Expand strings and yaml objects. |
| 109 |
class Expander: |
| 110 |
def __init__(self, config=None): |
| 111 |
if config: |
| 112 |
allow_empty_variables = config.yamlparser["allow_empty_variables"] |
| 113 |
else: |
| 114 |
allow_empty_variables = False |
| 115 |
_yaml_object_expanders = { |
| 116 |
cls: partial(call_expand, self) for cls in yaml_classes_list |
| 117 |
} |
| 118 |
self.expanders = { |
| 119 |
dict: partial(expand_dict, self), |
| 120 |
LocDict: partial(expand_dict, self), |
| 121 |
list: partial(expand_list, self), |
| 122 |
LocList: partial(expand_list, self), |
| 123 |
tuple: partial(expand_tuple, self), |
| 124 |
str: StrExpander(allow_empty_variables), |
| 125 |
LocString: StrExpander(allow_empty_variables), |
| 126 |
bool: dont_expand, |
| 127 |
int: dont_expand, |
| 128 |
float: dont_expand, |
| 129 |
type(None): dont_expand, |
| 130 |
**_yaml_object_expanders, |
| 131 |
} |
| 132 |
|
| 133 |
def expand(self, obj, params, key_pos=None, value_pos=None): |
| 134 |
t = type(obj) |
| 135 |
try: |
| 136 |
expander = self.expanders[t] |
| 137 |
except KeyError: |
| 138 |
raise JenkinsJobsException( |
| 139 |
f"Do not know how to expand type: {t!r}", pos=value_pos |
| 140 |
) |
| 141 |
return expander(obj, params, key_pos, value_pos) |
| 142 |
|
| 143 |
|
| 144 |
# Expand only yaml objects. |
| 145 |
class YamlObjectsExpander(Expander): |
| 146 |
def __init__(self, config=None): |
| 147 |
super().__init__(config) |
| 148 |
self.expanders.update( |
| 149 |
{ |
| 150 |
str: dont_expand, |
| 151 |
LocString: dont_expand, |
| 152 |
IncludeRawVerbatim: dont_expand, |
| 153 |
} |
| 154 |
) |
| 155 |
|
| 156 |
|
| 157 |
# Expand only string parameters. |
| 158 |
class StringsOnlyExpander(Expander): |
| 159 |
def __init__(self, config): |
| 160 |
super().__init__(config) |
| 161 |
_yaml_object_expanders = { |
| 162 |
cls: partial(dont_expand_yaml_object, self) for cls in yaml_classes_list |
| 163 |
} |
| 164 |
self.expanders.update(_yaml_object_expanders) |
| 165 |
|
| 166 |
|
| 167 |
def call_required_params(obj, pos): |
| 168 |
yield from obj.required_params |
| 169 |
|
| 170 |
|
| 171 |
def enum_dict_params(obj, pos): |
| 172 |
for key, value in obj.items(): |
| 173 |
yield from enum_required_params(key, obj.key_pos.get(key)) |
| 174 |
yield from enum_required_params(value, obj.value_pos.get(key)) |
| 175 |
|
| 176 |
|
| 177 |
def enum_seq_params(obj, pos): |
| 178 |
for idx, value in enumerate(obj): |
| 179 |
yield from enum_required_params(value, pos=None) |
| 180 |
|
| 181 |
|
| 182 |
def enum_loc_list_params(obj, pos): |
| 183 |
for idx, value in enumerate(obj): |
| 184 |
yield from enum_required_params(value, obj.value_pos[idx]) |
| 185 |
|
| 186 |
|
| 187 |
def no_parameters(obj, pos): |
| 188 |
return [] |
| 189 |
|
| 190 |
|
| 191 |
yaml_classes_enumers = {cls: call_required_params for cls in yaml_classes_list} |
| 192 |
|
| 193 |
param_enumers = { |
| 194 |
str: enum_str_format_required_params, |
| 195 |
LocString: enum_str_format_required_params, |
| 196 |
dict: enum_dict_params, |
| 197 |
LocDict: enum_dict_params, |
| 198 |
list: enum_seq_params, |
| 199 |
LocList: enum_loc_list_params, |
| 200 |
tuple: enum_seq_params, |
| 201 |
bool: no_parameters, |
| 202 |
int: no_parameters, |
| 203 |
float: no_parameters, |
| 204 |
type(None): no_parameters, |
| 205 |
**yaml_classes_enumers, |
| 206 |
} |
| 207 |
|
| 208 |
# Do not expand these. |
| 209 |
disable_expand_for = {"template-name"} |
| 210 |
|
| 211 |
|
| 212 |
def enum_required_params(obj, pos): |
| 213 |
t = type(obj) |
| 214 |
try: |
| 215 |
enumer = param_enumers[t] |
| 216 |
except KeyError: |
| 217 |
raise JenkinsJobsException( |
| 218 |
f"Do not know how to enumerate required parameters for type: {t!r}", |
| 219 |
pos=pos, |
| 220 |
) |
| 221 |
return enumer(obj, pos) |
| 222 |
|
| 223 |
|
| 224 |
def expand_parameters(expander, param_dict): |
| 225 |
expanded_params = LocDict() |
| 226 |
deps = {} # Variable name -> variable pos. |
| 227 |
|
| 228 |
def deps_context(): |
| 229 |
return [Context(f"Used by {n}", vp) for n, (kp, vp) in deps.items()] |
| 230 |
|
| 231 |
def expand(name): |
| 232 |
try: |
| 233 |
value = expanded_params[name] |
| 234 |
key_pos = expanded_params.key_pos.get(name) |
| 235 |
value_pos = expanded_params.value_pos.get(name) |
| 236 |
return (value, key_pos, value_pos) |
| 237 |
except KeyError: |
| 238 |
pass |
| 239 |
try: |
| 240 |
format = param_dict[name] |
| 241 |
except KeyError: |
| 242 |
return (StrictUndefined(name=name), None, None) |
| 243 |
key_pos = param_dict.key_pos.get(name) |
| 244 |
value_pos = param_dict.value_pos.get(name) |
| 245 |
if name in deps: |
| 246 |
expand_ctx = Context(f"While expanding {name!r}", key_pos) |
| 247 |
raise JenkinsJobsException( |
| 248 |
f"Recursive parameters usage: {' <- '.join(deps)}", |
| 249 |
pos=value_pos, |
| 250 |
ctx=[*deps_context(), expand_ctx], |
| 251 |
) |
| 252 |
if name in disable_expand_for: |
| 253 |
value = format |
| 254 |
else: |
| 255 |
required_params = list(enum_required_params(format, value_pos)) |
| 256 |
deps[name] = (key_pos, value_pos) |
| 257 |
try: |
| 258 |
params = LocDict.merge(expanded_params) |
| 259 |
for n in required_params: |
| 260 |
v, kp, vp = expand(n) |
| 261 |
params.set_item(n, v, kp, vp) |
| 262 |
finally: |
| 263 |
deps.popitem() |
| 264 |
try: |
| 265 |
value = expander.expand(format, params, key_pos, value_pos) |
| 266 |
except JenkinsJobsException as x: |
| 267 |
raise x.with_context( |
| 268 |
f"While expanding parameter {name!r}", |
| 269 |
pos=key_pos, |
| 270 |
ctx=deps_context(), |
| 271 |
) |
| 272 |
expanded_params.set_item(name, value, key_pos, value_pos) |
| 273 |
return (value, key_pos, value_pos) |
| 274 |
|
| 275 |
expand("name") # expand 'name' parameter first |
| 276 |
for name in filterfalse(lambda x: x == "name", param_dict): |
| 277 |
expand(name) |
| 278 |
return expanded_params |