Debian Codegraph @

jenkins_jobs/loader.py

raw
1 # Licensed under the Apache License, Version 2.0 (the "License");
2 # you may not use this file except in compliance with the License.
3 # You may obtain a copy of the License at
4 #
5 # http://www.apache.org/licenses/LICENSE-2.0
6 #
7 # Unless required by applicable law or agreed to in writing, software
8 # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
9 # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
10 # License for the specific language governing permissions and limitations
11 # under the License.
12 import functools
13 import io
14 import logging
15 import warnings
16 from functools import partial
17
18 from .errors import JenkinsJobsException
19 from .loc_loader import LocLoader
20 from .yaml_objects import BaseYamlObject
21 from .expander import YamlObjectsExpander, deprecated_yaml_tags, yaml_classes_list
22 from .roots import root_adders
23
24 logger = logging.getLogger(__name__)
25
26
27 class Loader(LocLoader):
28 @classmethod
29 def empty(cls, jjb_config):
30 return cls(io.StringIO(), jjb_config)
31
32 def __init__(
33 self, stream, jjb_config, source_path=None, source_dir=None, anchors=None
34 ):
35 super().__init__(stream, source_path)
36 self.jjb_config = jjb_config
37 self.source_path = source_path
38 self.source_dir = source_dir
39 self._retain_anchors = jjb_config.yamlparser["retain_anchors"]
40 if anchors:
41 # Override default set by super class.
42 self.anchors = anchors
43
44 # Override the default composer to skip resetting the anchors at the
45 # end of the current document.
46 def compose_document(self):
47 # Drop the DOCUMENT-START event.
48 self.get_event()
49 # Compose the root node.
50 node = self.compose_node(None, None)
51 # Drop the DOCUMENT-END event.
52 self.get_event()
53 return node
54
55 def _with_stream(self, stream, source_path, source_dir):
56 return Loader(stream, self.jjb_config, source_path, source_dir, self.anchors)
57
58 def load_fp(self, fp):
59 return self.load(fp)
60
61 # 1024 cache size seems like a reasonable balance between memory requirement and performance gain
62 @functools.lru_cache(maxsize=1024)
63 def load_path(self, path):
64 # Caching parsed file outputs is safe even with _retain_anchors set to True because:
65 # PyYAML does not allow updating anchor values, it is considered as anchor duplication
66 # So we can safely cache a parsed YAML for a file containing an alias since the alias can be defined
67 # only once and must be defined before use. The alias value will remain same irrespective of the number
68 # times a file is parsed
69 return self.load(path.read_text(), source_path=path, source_dir=path.parent)
70
71 def load(self, stream, source_path=None, source_dir=None):
72 loader = self._with_stream(stream, source_path, source_dir)
73 try:
74 return loader.get_single_data()
75 finally:
76 loader.dispose()
77 if self._retain_anchors:
78 self.anchors.update(loader.anchors)
79
80
81 def load_deprecated_yaml(tag, cls, loader, node):
82 warnings.warn(
83 f"Tag {tag!r} is deprecated, switch to using {cls.yaml_tag!r}",
84 UserWarning,
85 )
86 return cls.from_yaml(loader, node)
87
88
89 for cls in yaml_classes_list:
90 Loader.add_constructor(cls.yaml_tag, cls.from_yaml)
91
92 for tag, cls in deprecated_yaml_tags:
93 Loader.add_constructor(tag, partial(load_deprecated_yaml, tag, cls))
94
95
96 def is_stdin(path):
97 return hasattr(path, "read")
98
99
100 def enum_expanded_paths(path_list):
101 visited_set = set()
102
103 def real(path):
104 real_path = path.resolve()
105 if real_path in visited_set:
106 logger.warning(
107 "File '%s' is already added as '%s'; ignoring reference to avoid"
108 " duplicating YAML definitions.",
109 path,
110 real_path,
111 )
112 else:
113 yield real_path
114 visited_set.add(real_path)
115
116 for path in path_list:
117 if is_stdin(path):
118 yield path
119 elif path.is_dir():
120 for p in path.iterdir():
121 if p.suffix in {".yml", ".yaml"}:
122 yield from real(p)
123 else:
124 yield from real(path)
125
126
127 def load_files(config, roots, path_list):
128 expander = YamlObjectsExpander(config)
129 loader = Loader.empty(config)
130 for path in enum_expanded_paths(path_list):
131 if is_stdin(path):
132 data = loader.load_fp(path)
133 else:
134 data = loader.load_path(path)
135 if data is None:
136 continue
137 if not isinstance(data, list):
138 raise JenkinsJobsException(
139 f"The topmost collection must be a list, but is: {data}",
140 pos=data.pos,
141 )
142 for idx, item in enumerate(data):
143 if not isinstance(item, dict):
144 raise JenkinsJobsException(
145 f"Topmost list should contain single-item dict,"
146 f" not a {type(item)}. Missing indent?",
147 pos=data.value_pos[idx],
148 )
149 if len(item) != 1:
150 raise JenkinsJobsException(
151 f"Topmost dict should be single-item,"
152 f" but have keys {list(item.keys())}. Missing indent?",
153 pos=item.pos,
154 )
155 kind, contents = next(iter(item.items()))
156 if kind.startswith("_"):
157 continue
158 if isinstance(contents, BaseYamlObject):
159 contents = contents.expand(expander, params={})
160 try:
161 adder = root_adders[kind]
162 except KeyError:
163 raise JenkinsJobsException(
164 f"Unknown topmost element type : {kind!r};"
165 f" known are: {','.join(root_adders)}.",
166 pos=item.pos,
167 )
168 adder(config, roots, contents, item.pos)