Debian Codegraph @

data/generators/generate-cmake-syntax.py

raw
1 #!/usr/bin/env python3
2 # -*- coding: utf-8 -*-
3 #
4 # Generate Kate syntax file for CMake
5 #
6 # SPDX-FileCopyrightText: 2017-2020 Alex Turbov <[email protected]>
7 #
8 # To install prerequisites:
9 #
10 # $ pip install --user click jinja2 pyyaml
11 #
12 # To use:
13 #
14 # $ ./generate-cmake-syntax.py cmake.yaml > ../syntax/cmake.xml
15 #
16 import click
17 import jinja2
18 import re
19 import yaml
20
21 from lxml import etree
22
23
24 _TEMPLATED_NAME = re.compile('<[^>]+>')
25 _PROPERTY_KEYS = [
26 'global-properties'
27 , 'directory-properties'
28 , 'target-properties'
29 , 'source-properties'
30 , 'test-properties'
31 , 'cache-properties'
32 , 'install-properties'
33 ]
34 _KW_RE_LIST = ['kw', 're']
35 _VAR_KIND_LIST = ['variables', 'deprecated-or-internal-variables', 'environment-variables']
36 _CONTROL_FLOW_LIST = set((
37 'break'
38 , 'continue'
39 , 'elseif'
40 , 'else'
41 , 'endforeach'
42 , 'endif'
43 , 'endwhile'
44 , 'foreach'
45 , 'if'
46 , 'return'
47 , 'while'
48 ))
49
50
51 def try_transform_placeholder_string_to_regex(name):
52 '''
53 NOTE Some placeholders are not IDs, but numbers...
54 `CMAKE_MATCH_<N>` 4 example
55 '''
56 m = _TEMPLATED_NAME.split(name)
57 if 'CMAKE_MATCH_' in m:
58 return 'CMAKE_MATCH_[0-9]+'
59
60 if 'CMAKE_ARGV' in m:
61 return 'CMAKE_ARGV[0-9]+'
62
63 if 'CMAKE_POLICY_DEFAULT_CMP' in m:
64 return 'CMAKE_POLICY_DEFAULT_CMP[0-9]{4}'
65
66 if 'CMAKE_POLICY_WARNING_CMP' in m:
67 return 'CMAKE_POLICY_WARNING_CMP[0-9]{4}'
68
69 if 'ARGV' in m:
70 return 'ARGV[0-9]+'
71
72 return '&var_ref_re;'.join(m) if 1 < len(m) else name
73
74
75 def try_placeholders_to_regex(names):
76 if not names:
77 return None
78 l = map(try_transform_placeholder_string_to_regex, names)
79 l = sorted(l, reverse=True)
80 return '\\b(?:' + '|'.join(l) + ')\\b'
81
82
83 def partition_iterable(fn, iterable):
84 true, false = [], []
85 for i in iterable:
86 (false, true)[int(fn(i))].append(i)
87 return true, false
88
89
90 def _transform_command_set(cmd, list_name):
91 args, args_re = partition_iterable(lambda x: _TEMPLATED_NAME.search(x) is None, cmd[list_name])
92 del cmd[list_name]
93 list_name = list_name.replace('-', '_')
94
95 cmd[list_name] = {k: sorted(set(v)) for k, v in zip(_KW_RE_LIST, [args, args_re])}
96 cmd[list_name]['re'] = try_placeholders_to_regex(args_re)
97
98 return cmd
99
100
101 def transform_command(cmd):
102 can_be_nulary = True
103
104 if 'name' not in cmd:
105 raise RuntimeError('Command have no name')
106
107 if 'named-args' in cmd:
108 new_cmd = _transform_command_set(cmd, 'named-args')
109 assert new_cmd == cmd
110 can_be_nulary = False
111
112 if 'special-args' in cmd:
113 new_cmd = _transform_command_set(cmd, 'special-args')
114 assert new_cmd == cmd
115 can_be_nulary = False
116
117 if 'property-args' in cmd:
118 new_cmd = _transform_command_set(cmd, 'property-args')
119 assert new_cmd == cmd
120 can_be_nulary = False
121
122 cmd['nested_parentheses'] = cmd.get('nested-parentheses?', False)
123
124 if 'first-arg-is-target?' in cmd:
125 cmd['first_arg_is_target'] = cmd['first-arg-is-target?']
126 can_be_nulary = False
127
128 if 'first-args-are-targets?' in cmd:
129 cmd['first_args_are_targets'] = cmd['first-args-are-targets?']
130 can_be_nulary = False
131
132 if 'has-target-name-after-kw' in cmd:
133 cmd['has_target_name_after_kw'] = cmd['has-target-name-after-kw']
134 can_be_nulary = False
135
136 if 'has-target-names-after-kw' in cmd:
137 cmd['has_target_names_after_kw'] = cmd['has-target-names-after-kw']
138 can_be_nulary = False
139
140 if 'second-arg-is-target?' in cmd:
141 cmd['second_arg_is_target'] = cmd['second-arg-is-target?']
142 can_be_nulary = False
143
144 if 'nulary?' in cmd and cmd['nulary?'] and not can_be_nulary:
145 raise RuntimeError('Command `{}` w/ args declared nulary!?'.format(cmd['name']))
146
147 if 'start-region' in cmd:
148 cmd['start_region'] = cmd['start-region']
149
150 if 'end-region' in cmd:
151 cmd['end_region'] = cmd['end-region']
152
153 cmd['attribute'] = 'Control Flow' if cmd['name'] in _CONTROL_FLOW_LIST else 'Command'
154
155 return cmd
156
157
158 def remove_duplicate_list_nodes(contexts, highlighting):
159 remap = {}
160
161 items_by_kws = {}
162 # extract duplicate keyword list
163 for items in highlighting:
164 if items.tag != 'list':
165 break
166 k = '<'.join(item.text for item in items)
167 name = items.attrib['name']
168 rename = items_by_kws.get(k)
169 if rename:
170 remap[name] = rename
171 highlighting.remove(items)
172 else:
173 items_by_kws[k] = name
174
175 # update keyword list name referenced by each rule
176 for context in contexts:
177 for rule in context:
178 if rule.tag == 'keyword':
179 name = rule.attrib['String']
180 rule.attrib['String'] = remap.get(name, name)
181
182
183 def remove_duplicate_context_nodes(contexts):
184 # 3 levels: ctx, ctx_op and ctx_op_nested
185 for _ in range(3):
186 remap = {}
187 duplicated = {}
188
189 # remove duplicate nodes
190 for context in contexts:
191 name = context.attrib['name']
192 context.attrib['name'] = 'dummy'
193 ref = duplicated.setdefault(etree.tostring(context), [])
194 if ref:
195 contexts.remove(context)
196 else:
197 context.attrib['name'] = name
198 ref.append(name)
199 remap[name] = ref[0]
200
201 # update context name referenced by each rule
202 for context in contexts:
203 for rule in context:
204 ref = remap.get(rule.attrib.get('context'))
205 if ref:
206 rule.attrib['context'] = ref
207
208
209 def remove_duplicate_nodes(xml_string):
210 parser = etree.XMLParser(resolve_entities=False, collect_ids=False)
211 root = etree.fromstring(xml_string.encode(), parser=parser)
212 highlighting = root[0]
213
214 contexts = highlighting.find('contexts')
215
216 remove_duplicate_list_nodes(contexts, highlighting)
217 remove_duplicate_context_nodes(contexts)
218
219 # reformat comments
220 xml = etree.tostring(root)
221 xml = re.sub(b'(?=[^\n ])<!--', b'\n<!--', xml)
222 xml = re.sub(b'-->(?=[^ \n])', b'-->\n', xml)
223
224 # extract DOCTYPE removed by etree.fromstring and reformat <language>
225 doctype = xml_string[:xml_string.find('<highlighting')]
226
227 # remove unformatted <language>
228 xml = xml[xml.find(b'<highlighting'):]
229
230 # last comment removed by etree.fromstring
231 last_comment = '\n<!-- kate: replace-tabs on; indent-width 2; tab-width 2; -->'
232
233 return f'{doctype}{xml.decode()}{last_comment}'
234
235
236 #BEGIN Jinja filters
237
238 def cmd_is_nulary(cmd):
239 return cmd.setdefault('nulary?', False)
240
241 #END Jinja filters
242
243
244 @click.command()
245 @click.argument('input_yaml', type=click.File('r'))
246 @click.argument('template', type=click.File('r'), default='./cmake.xml.tpl')
247 def cli(input_yaml, template):
248 data = yaml.load(input_yaml, Loader=yaml.BaseLoader)
249
250 # Partition `variables` and `environment-variables` lists into "pure" (key)words and regexes to match
251 for var_key in _VAR_KIND_LIST:
252 data[var_key] = {
253 k: sorted(set(v)) for k, v in zip(
254 _KW_RE_LIST
255 , [*partition_iterable(lambda x: _TEMPLATED_NAME.search(x) is None, data[var_key])]
256 )
257 }
258 data[var_key]['re'] = try_placeholders_to_regex(data[var_key]['re'])
259
260 # Transform properties and make all-properties list
261 data['properties'] = {}
262 for prop in _PROPERTY_KEYS:
263 python_prop_list_name = prop.replace('-', '_')
264 props, props_re = partition_iterable(lambda x: _TEMPLATED_NAME.search(x) is None, data[prop])
265 del data[prop]
266
267 data['properties'][python_prop_list_name] = {
268 k: sorted(set(v)) for k, v in zip(_KW_RE_LIST, [props, props_re])
269 }
270 data['properties'][python_prop_list_name]['re'] = try_placeholders_to_regex(props_re)
271
272 data['properties']['kinds'] = list(map(lambda name: name.replace('-', '_'), _PROPERTY_KEYS))
273
274 # Make all commands list
275 data['commands'] = list(
276 map(
277 transform_command
278 , data['scripting-commands'] + data['project-commands'] + data['ctest-commands']
279 )
280 )
281 data['standard_module_commands'] = list(
282 map(
283 transform_command
284 , data['standard-module-commands']
285 )
286 )
287 del data['standard-module-commands']
288
289 # Fix node names to be accessible from Jinja template
290 data['generator_expressions'] = data['generator-expressions']
291 data['deprecated_or_internal_variables'] = data['deprecated-or-internal-variables']
292 data['environment_variables'] = data['environment-variables']
293 del data['generator-expressions']
294 del data['deprecated-or-internal-variables']
295 del data['environment-variables']
296
297 env = jinja2.Environment(
298 keep_trailing_newline=True
299 )
300 env.block_start_string = '<!--['
301 env.block_end_string = ']-->'
302 env.variable_start_string = '<!--{'
303 env.variable_end_string = '}-->'
304 env.comment_start_string = '<!--#'
305 env.comment_end_string = '#-->'
306
307 # Register convenience filters
308 env.tests['nulary'] = cmd_is_nulary
309
310 tpl = env.from_string(template.read())
311 result = tpl.render(data)
312 result = remove_duplicate_nodes(result)
313
314 print(result)
315
316
317 if __name__ == '__main__':
318 cli()
319 # TODO Handle execptions and show errors