Debian Codegraph @

Tools/clinic/libclinic/block_parser.py

raw
1 from __future__ import annotations
2 import collections
3 import dataclasses as dc
4 import re
5 import shlex
6 from typing import Any
7
8 import libclinic
9 from libclinic import fail, ClinicError
10 from libclinic.language import Language
11 from libclinic.function import (
12 Module, Class, Function)
13
14
15 @dc.dataclass(slots=True, repr=False)
16 class Block:
17 r"""
18 Represents a single block of text embedded in
19 another file. If dsl_name is None, the block represents
20 verbatim text, raw original text from the file, in
21 which case "input" will be the only non-false member.
22 If dsl_name is not None, the block represents a Clinic
23 block.
24
25 input is always str, with embedded \n characters.
26 input represents the original text from the file;
27 if it's a Clinic block, it is the original text with
28 the body_prefix and redundant leading whitespace removed.
29
30 dsl_name is either str or None. If str, it's the text
31 found on the start line of the block between the square
32 brackets.
33
34 signatures is a list.
35 It may only contain clinic.Module, clinic.Class, and
36 clinic.Function objects. At the moment it should
37 contain at most one of each.
38
39 output is either str or None. If str, it's the output
40 from this block, with embedded '\n' characters.
41
42 indent is a str. It's the leading whitespace
43 that was found on every line of input. (If body_prefix is
44 not empty, this is the indent *after* removing the
45 body_prefix.)
46
47 "indent" is different from the concept of "preindent"
48 (which is not stored as state on Block objects).
49 "preindent" is the whitespace that
50 was found in front of every line of input *before* the
51 "body_prefix" (see the Language object). If body_prefix
52 is empty, preindent must always be empty too.
53
54 To illustrate the difference between "indent" and "preindent":
55
56 Assume that '_' represents whitespace.
57 If the block processed was in a Python file, and looked like this:
58 ____#/*[python]
59 ____#__for a in range(20):
60 ____#____print(a)
61 ____#[python]*/
62 "preindent" would be "____" and "indent" would be "__".
63
64 """
65 input: str
66 dsl_name: str | None = None
67 signatures: list[Module | Class | Function] = dc.field(default_factory=list)
68 output: Any = None # TODO: Very dynamic; probably untypeable in its current form?
69 indent: str = ''
70
71 def __repr__(self) -> str:
72 dsl_name = self.dsl_name or "text"
73 def summarize(s: object) -> str:
74 s = repr(s)
75 if len(s) > 30:
76 return s[:26] + "..." + s[0]
77 return s
78 parts = (
79 repr(dsl_name),
80 f"input={summarize(self.input)}",
81 f"output={summarize(self.output)}"
82 )
83 return f"<clinic.Block {' '.join(parts)}>"
84
85
86 class BlockParser:
87 """
88 Block-oriented parser for Argument Clinic.
89 Iterator, yields Block objects.
90 """
91
92 def __init__(
93 self,
94 input: str,
95 language: Language,
96 *,
97 verify: bool = True
98 ) -> None:
99 """
100 "input" should be a str object
101 with embedded \n characters.
102
103 "language" should be a Language object.
104 """
105 language.validate()
106
107 self.input = collections.deque(reversed(input.splitlines(keepends=True)))
108 self.block_start_line_number = self.line_number = 0
109
110 self.language = language
111 before, _, after = language.start_line.partition('{dsl_name}')
112 assert _ == '{dsl_name}'
113 self.find_start_re = libclinic.create_regex(before, after,
114 whole_line=False)
115 self.start_re = libclinic.create_regex(before, after)
116 self.verify = verify
117 self.last_checksum_re: re.Pattern[str] | None = None
118 self.last_dsl_name: str | None = None
119 self.dsl_name: str | None = None
120 self.first_block = True
121
122 def __iter__(self) -> BlockParser:
123 return self
124
125 def __next__(self) -> Block:
126 while True:
127 if not self.input:
128 raise StopIteration
129
130 if self.dsl_name:
131 try:
132 return_value = self.parse_clinic_block(self.dsl_name)
133 except ClinicError as exc:
134 exc.filename = self.language.filename
135 exc.lineno = self.line_number
136 raise
137 self.dsl_name = None
138 self.first_block = False
139 return return_value
140 block = self.parse_verbatim_block()
141 if self.first_block and not block.input:
142 continue
143 self.first_block = False
144 return block
145
146
147 def is_start_line(self, line: str) -> str | None:
148 match = self.start_re.match(line.lstrip())
149 return match.group(1) if match else None
150
151 def _line(self, lookahead: bool = False) -> str:
152 self.line_number += 1
153 line = self.input.pop()
154 if not lookahead:
155 self.language.parse_line(line)
156 return line
157
158 def parse_verbatim_block(self) -> Block:
159 lines = []
160 self.block_start_line_number = self.line_number
161
162 while self.input:
163 line = self._line()
164 dsl_name = self.is_start_line(line)
165 if dsl_name:
166 self.dsl_name = dsl_name
167 break
168 lines.append(line)
169
170 return Block("".join(lines))
171
172 def parse_clinic_block(self, dsl_name: str) -> Block:
173 in_lines = []
174 self.block_start_line_number = self.line_number + 1
175 stop_line = self.language.stop_line.format(dsl_name=dsl_name)
176 body_prefix = self.language.body_prefix.format(dsl_name=dsl_name)
177
178 def is_stop_line(line: str) -> bool:
179 # make sure to recognize stop line even if it
180 # doesn't end with EOL (it could be the very end of the file)
181 if line.startswith(stop_line):
182 remainder = line.removeprefix(stop_line)
183 if remainder and not remainder.isspace():
184 fail(f"Garbage after stop line: {remainder!r}")
185 return True
186 else:
187 # gh-92256: don't allow incorrectly formatted stop lines
188 if line.lstrip().startswith(stop_line):
189 fail(f"Whitespace is not allowed before the stop line: {line!r}")
190 return False
191
192 # consume body of program
193 while self.input:
194 line = self._line()
195 if is_stop_line(line) or self.is_start_line(line):
196 break
197 if body_prefix:
198 line = line.lstrip()
199 assert line.startswith(body_prefix)
200 line = line.removeprefix(body_prefix)
201 in_lines.append(line)
202
203 # consume output and checksum line, if present.
204 if self.last_dsl_name == dsl_name:
205 checksum_re = self.last_checksum_re
206 else:
207 before, _, after = self.language.checksum_line.format(dsl_name=dsl_name, arguments='{arguments}').partition('{arguments}')
208 assert _ == '{arguments}'
209 checksum_re = libclinic.create_regex(before, after, word=False)
210 self.last_dsl_name = dsl_name
211 self.last_checksum_re = checksum_re
212 assert checksum_re is not None
213
214 # scan forward for checksum line
215 out_lines = []
216 arguments = None
217 while self.input:
218 line = self._line(lookahead=True)
219 match = checksum_re.match(line.lstrip())
220 arguments = match.group(1) if match else None
221 if arguments:
222 break
223 out_lines.append(line)
224 if self.is_start_line(line):
225 break
226
227 output: str | None
228 output = "".join(out_lines)
229 if arguments:
230 d = {}
231 for field in shlex.split(arguments):
232 name, equals, value = field.partition('=')
233 if not equals:
234 fail(f"Mangled Argument Clinic marker line: {line!r}")
235 d[name.strip()] = value.strip()
236
237 if self.verify:
238 if 'input' in d:
239 checksum = d['output']
240 else:
241 checksum = d['checksum']
242
243 computed = libclinic.compute_checksum(output, len(checksum))
244 if checksum != computed:
245 fail("Checksum mismatch! "
246 f"Expected {checksum!r}, computed {computed!r}. "
247 "Suggested fix: remove all generated code including "
248 "the end marker, or use the '-f' option.")
249 else:
250 # put back output
251 output_lines = output.splitlines(keepends=True)
252 self.line_number -= len(output_lines)
253 self.input.extend(reversed(output_lines))
254 output = None
255
256 return Block("".join(in_lines), dsl_name, output=output)