Debian Codegraph @

Doc/tools/extensions/grammar_snippet.py

raw
1 """Support for documenting Python's grammar."""
2
3 from __future__ import annotations
4
5 import re
6 from typing import TYPE_CHECKING
7
8 from docutils import nodes
9 from docutils.parsers.rst import directives
10 from sphinx import addnodes
11 from sphinx.domains.std import token_xrefs
12 from sphinx.util.docutils import SphinxDirective
13 from sphinx.util.nodes import make_id
14
15 if TYPE_CHECKING:
16 from collections.abc import Iterable, Iterator, Sequence
17 from typing import Any, Final
18
19 from docutils.nodes import Node
20 from sphinx.application import Sphinx
21 from sphinx.util.typing import ExtensionMetadata
22
23
24 class snippet_string_node(nodes.inline): # noqa: N801 (snake_case is fine)
25 """Node for a string literal in a grammar snippet."""
26
27 def __init__(
28 self,
29 rawsource: str = '',
30 text: str = '',
31 *children: Node,
32 **attributes: Any,
33 ) -> None:
34 super().__init__(rawsource, text, *children, **attributes)
35 # Use the Pygments highlight class for `Literal.String.Other`
36 self['classes'].append('sx')
37
38
39 class GrammarSnippetBase(SphinxDirective):
40 """Common functionality for GrammarSnippetDirective & CompatProductionList."""
41
42 # The option/argument handling is left to the individual classes.
43
44 grammar_re: Final = re.compile(
45 r"""
46 (?P<rule_name>^[a-zA-Z0-9_]+) # identifier at start of line
47 (?=:) # ... followed by a colon
48 |
49 (?P<rule_ref>`[^\s`]+`) # identifier in backquotes
50 |
51 (?P<single_quoted>'[^']*') # string in 'quotes'
52 |
53 (?P<double_quoted>"[^"]*") # string in "quotes"
54 """,
55 re.VERBOSE,
56 )
57
58 def make_grammar_snippet(
59 self, options: dict[str, Any], content: Sequence[str]
60 ) -> list[addnodes.productionlist]:
61 """Create a literal block from options & content."""
62
63 group_name = options['group']
64 node_location = self.get_location()
65 production_nodes = []
66 for rawsource, production_defs in self.production_definitions(content):
67 production = self.make_production(
68 rawsource,
69 production_defs,
70 group_name=group_name,
71 location=node_location,
72 )
73 production_nodes.append(production)
74
75 node = addnodes.productionlist(
76 '',
77 *production_nodes,
78 support_smartquotes=False,
79 classes=['highlight'],
80 )
81 self.set_source_info(node)
82 return [node]
83
84 def production_definitions(
85 self, lines: Iterable[str], /
86 ) -> Iterator[tuple[str, list[tuple[str, str]]]]:
87 """Yield pairs of rawsource and production content dicts."""
88 production_lines: list[str] = []
89 production_content: list[tuple[str, str]] = []
90 for line in lines:
91 # If this line is the start of a new rule (text in the column 1),
92 # emit the current production and start a new one.
93 if not line[:1].isspace():
94 rawsource = '\n'.join(production_lines)
95 production_lines.clear()
96 if production_content:
97 yield rawsource, production_content
98 production_content = []
99
100 # Append the current line for the raw source
101 production_lines.append(line)
102
103 # Parse the line into constituent parts
104 last_pos = 0
105 for match in self.grammar_re.finditer(line):
106 # Handle text between matches
107 if match.start() > last_pos:
108 unmatched_text = line[last_pos : match.start()]
109 production_content.append(('text', unmatched_text))
110 last_pos = match.end()
111
112 # Handle matches.
113 # After filtering None (non-matches), exactly one groupdict()
114 # entry should remain.
115 [(re_group_name, content)] = (
116 (re_group_name, content)
117 for re_group_name, content in match.groupdict().items()
118 if content is not None
119 )
120 production_content.append((re_group_name, content))
121 production_content.append(('text', line[last_pos:] + '\n'))
122
123 # Emit the final production
124 if production_content:
125 rawsource = '\n'.join(production_lines)
126 yield rawsource, production_content
127
128 def make_production(
129 self,
130 rawsource: str,
131 production_defs: list[tuple[str, str]],
132 *,
133 group_name: str,
134 location: str,
135 ) -> addnodes.production:
136 """Create a production node from a list of parts."""
137 production_node = addnodes.production(rawsource)
138 for re_group_name, content in production_defs:
139 match re_group_name:
140 case 'rule_name':
141 production_node += self.make_name_target(
142 name=content,
143 production_group=group_name,
144 location=location,
145 )
146 case 'rule_ref':
147 production_node += token_xrefs(content, group_name)
148 case 'single_quoted' | 'double_quoted':
149 production_node += snippet_string_node('', content)
150 case 'text':
151 production_node += nodes.Text(content)
152 case _:
153 raise ValueError(f'unhandled match: {re_group_name!r}')
154 return production_node
155
156 def make_name_target(
157 self,
158 *,
159 name: str,
160 production_group: str,
161 location: str,
162 ) -> addnodes.literal_strong:
163 """Make a link target for the given production."""
164
165 # Cargo-culted magic to make `name_node` a link target
166 # similar to Sphinx `production`.
167 # This needs to be the same as what Sphinx does
168 # to avoid breaking existing links.
169
170 name_node = addnodes.literal_strong(name, name)
171 prefix = f'grammar-token-{production_group}'
172 node_id = make_id(self.env, self.state.document, prefix, name)
173 name_node['ids'].append(node_id)
174 self.state.document.note_implicit_target(name_node, name_node)
175 obj_name = f'{production_group}:{name}' if production_group else name
176 std = self.env.domains.standard_domain
177 std.note_object('token', obj_name, node_id, location=location)
178 return name_node
179
180
181 class GrammarSnippetDirective(GrammarSnippetBase):
182 """Transform a grammar-snippet directive to a Sphinx literal_block
183
184 That is, turn something like:
185
186 .. grammar-snippet:: file
187 :group: python-grammar
188
189 file: (NEWLINE | statement)*
190
191 into something similar to Sphinx productionlist, but better suited
192 for our needs:
193 - Instead of `::=`, use a colon, as in `Grammar/python.gram`
194 - Show the listing almost as is, with no auto-alignment.
195 The only special character is the backtick, which marks tokens.
196
197 Unlike Sphinx's productionlist, this directive supports options.
198 The "group" must be given as a named option.
199 The content must be preceded by a blank line (like with most ReST
200 directives).
201 """
202
203 has_content = True
204 option_spec = {
205 'group': directives.unchanged_required,
206 }
207
208 # We currently ignore arguments.
209 required_arguments = 0
210 optional_arguments = 1
211 final_argument_whitespace = True
212
213 def run(self) -> list[addnodes.productionlist]:
214 return self.make_grammar_snippet(self.options, self.content)
215
216
217 class CompatProductionList(GrammarSnippetBase):
218 """Create grammar snippets from reST productionlist syntax
219
220 This is intended to be a transitional directive, used while we switch
221 from productionlist to grammar-snippet.
222 It makes existing docs that use the ReST syntax look like grammar-snippet,
223 as much as possible.
224 """
225
226 has_content = False
227 required_arguments = 1
228 optional_arguments = 0
229 final_argument_whitespace = True
230 option_spec = {}
231
232 def run(self) -> list[addnodes.productionlist]:
233 # The "content" of a productionlist is actually the first and only
234 # argument. The first line is the group; the rest is the content lines.
235 lines = self.arguments[0].splitlines()
236 group = lines[0].strip()
237 options = {'group': group}
238 # We assume there's a colon in each line; align on it.
239 align_column = max(line.index(':') for line in lines[1:]) + 1
240 content = []
241 for line in lines[1:]:
242 rule_name, _colon, text = line.partition(':')
243 rule_name = rule_name.strip()
244 if rule_name:
245 name_part = rule_name + ':'
246 else:
247 name_part = ''
248 content.append(f'{name_part:<{align_column}}{text}')
249 return self.make_grammar_snippet(options, content)
250
251
252 def setup(app: Sphinx) -> ExtensionMetadata:
253 app.add_directive('grammar-snippet', GrammarSnippetDirective)
254 app.add_directive_to_domain(
255 'std', 'productionlist', CompatProductionList, override=True
256 )
257 return {
258 'version': '1.0',
259 'parallel_read_safe': True,
260 'parallel_write_safe': True,
261 }