| 1 |
# Licensed under the Apache License, Version 2.0 (the "License"); |
| 2 |
# you may not use this file except in compliance with the License. |
| 3 |
# You may obtain a copy of the License at |
| 4 |
# |
| 5 |
# http://www.apache.org/licenses/LICENSE-2.0 |
| 6 |
# |
| 7 |
# Unless required by applicable law or agreed to in writing, software |
| 8 |
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT |
| 9 |
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the |
| 10 |
# License for the specific language governing permissions and limitations |
| 11 |
# under the License. |
| 12 |
import functools |
| 13 |
import io |
| 14 |
import logging |
| 15 |
import warnings |
| 16 |
from functools import partial |
| 17 |
|
| 18 |
from .errors import JenkinsJobsException |
| 19 |
from .loc_loader import LocLoader |
| 20 |
from .yaml_objects import BaseYamlObject |
| 21 |
from .expander import YamlObjectsExpander, deprecated_yaml_tags, yaml_classes_list |
| 22 |
from .roots import root_adders |
| 23 |
|
| 24 |
logger = logging.getLogger(__name__) |
| 25 |
|
| 26 |
|
| 27 |
class Loader(LocLoader): |
| 28 |
@classmethod |
| 29 |
def empty(cls, jjb_config): |
| 30 |
return cls(io.StringIO(), jjb_config) |
| 31 |
|
| 32 |
def __init__( |
| 33 |
self, stream, jjb_config, source_path=None, source_dir=None, anchors=None |
| 34 |
): |
| 35 |
super().__init__(stream, source_path) |
| 36 |
self.jjb_config = jjb_config |
| 37 |
self.source_path = source_path |
| 38 |
self.source_dir = source_dir |
| 39 |
self._retain_anchors = jjb_config.yamlparser["retain_anchors"] |
| 40 |
if anchors: |
| 41 |
# Override default set by super class. |
| 42 |
self.anchors = anchors |
| 43 |
|
| 44 |
# Override the default composer to skip resetting the anchors at the |
| 45 |
# end of the current document. |
| 46 |
def compose_document(self): |
| 47 |
# Drop the DOCUMENT-START event. |
| 48 |
self.get_event() |
| 49 |
# Compose the root node. |
| 50 |
node = self.compose_node(None, None) |
| 51 |
# Drop the DOCUMENT-END event. |
| 52 |
self.get_event() |
| 53 |
return node |
| 54 |
|
| 55 |
def _with_stream(self, stream, source_path, source_dir): |
| 56 |
return Loader(stream, self.jjb_config, source_path, source_dir, self.anchors) |
| 57 |
|
| 58 |
def load_fp(self, fp): |
| 59 |
return self.load(fp) |
| 60 |
|
| 61 |
# 1024 cache size seems like a reasonable balance between memory requirement and performance gain |
| 62 |
@functools.lru_cache(maxsize=1024) |
| 63 |
def load_path(self, path): |
| 64 |
# Caching parsed file outputs is safe even with _retain_anchors set to True because: |
| 65 |
# PyYAML does not allow updating anchor values, it is considered as anchor duplication |
| 66 |
# So we can safely cache a parsed YAML for a file containing an alias since the alias can be defined |
| 67 |
# only once and must be defined before use. The alias value will remain same irrespective of the number |
| 68 |
# times a file is parsed |
| 69 |
return self.load(path.read_text(), source_path=path, source_dir=path.parent) |
| 70 |
|
| 71 |
def load(self, stream, source_path=None, source_dir=None): |
| 72 |
loader = self._with_stream(stream, source_path, source_dir) |
| 73 |
try: |
| 74 |
return loader.get_single_data() |
| 75 |
finally: |
| 76 |
loader.dispose() |
| 77 |
if self._retain_anchors: |
| 78 |
self.anchors.update(loader.anchors) |
| 79 |
|
| 80 |
|
| 81 |
def load_deprecated_yaml(tag, cls, loader, node): |
| 82 |
warnings.warn( |
| 83 |
f"Tag {tag!r} is deprecated, switch to using {cls.yaml_tag!r}", |
| 84 |
UserWarning, |
| 85 |
) |
| 86 |
return cls.from_yaml(loader, node) |
| 87 |
|
| 88 |
|
| 89 |
for cls in yaml_classes_list: |
| 90 |
Loader.add_constructor(cls.yaml_tag, cls.from_yaml) |
| 91 |
|
| 92 |
for tag, cls in deprecated_yaml_tags: |
| 93 |
Loader.add_constructor(tag, partial(load_deprecated_yaml, tag, cls)) |
| 94 |
|
| 95 |
|
| 96 |
def is_stdin(path): |
| 97 |
return hasattr(path, "read") |
| 98 |
|
| 99 |
|
| 100 |
def enum_expanded_paths(path_list): |
| 101 |
visited_set = set() |
| 102 |
|
| 103 |
def real(path): |
| 104 |
real_path = path.resolve() |
| 105 |
if real_path in visited_set: |
| 106 |
logger.warning( |
| 107 |
"File '%s' is already added as '%s'; ignoring reference to avoid" |
| 108 |
" duplicating YAML definitions.", |
| 109 |
path, |
| 110 |
real_path, |
| 111 |
) |
| 112 |
else: |
| 113 |
yield real_path |
| 114 |
visited_set.add(real_path) |
| 115 |
|
| 116 |
for path in path_list: |
| 117 |
if is_stdin(path): |
| 118 |
yield path |
| 119 |
elif path.is_dir(): |
| 120 |
for p in path.iterdir(): |
| 121 |
if p.suffix in {".yml", ".yaml"}: |
| 122 |
yield from real(p) |
| 123 |
else: |
| 124 |
yield from real(path) |
| 125 |
|
| 126 |
|
| 127 |
def load_files(config, roots, path_list): |
| 128 |
expander = YamlObjectsExpander(config) |
| 129 |
loader = Loader.empty(config) |
| 130 |
for path in enum_expanded_paths(path_list): |
| 131 |
if is_stdin(path): |
| 132 |
data = loader.load_fp(path) |
| 133 |
else: |
| 134 |
data = loader.load_path(path) |
| 135 |
if data is None: |
| 136 |
continue |
| 137 |
if not isinstance(data, list): |
| 138 |
raise JenkinsJobsException( |
| 139 |
f"The topmost collection must be a list, but is: {data}", |
| 140 |
pos=data.pos, |
| 141 |
) |
| 142 |
for idx, item in enumerate(data): |
| 143 |
if not isinstance(item, dict): |
| 144 |
raise JenkinsJobsException( |
| 145 |
f"Topmost list should contain single-item dict," |
| 146 |
f" not a {type(item)}. Missing indent?", |
| 147 |
pos=data.value_pos[idx], |
| 148 |
) |
| 149 |
if len(item) != 1: |
| 150 |
raise JenkinsJobsException( |
| 151 |
f"Topmost dict should be single-item," |
| 152 |
f" but have keys {list(item.keys())}. Missing indent?", |
| 153 |
pos=item.pos, |
| 154 |
) |
| 155 |
kind, contents = next(iter(item.items())) |
| 156 |
if kind.startswith("_"): |
| 157 |
continue |
| 158 |
if isinstance(contents, BaseYamlObject): |
| 159 |
contents = contents.expand(expander, params={}) |
| 160 |
try: |
| 161 |
adder = root_adders[kind] |
| 162 |
except KeyError: |
| 163 |
raise JenkinsJobsException( |
| 164 |
f"Unknown topmost element type : {kind!r};" |
| 165 |
f" known are: {','.join(root_adders)}.", |
| 166 |
pos=item.pos, |
| 167 |
) |
| 168 |
adder(config, roots, contents, item.pos) |