Debian Codegraph @

ptk/lexer.py

raw
1 # -*- coding: utf-8 -*-
2
3 # (c) Jérôme Laheurte 2015-2026
4 # See LICENSE.txt
5
6 import inspect
7 import re
8 import collections
9 import warnings
10
11 from ptk.utils import Singleton, callback_by_name, chars
12
13
14 LexerPosition = collections.namedtuple('_LexerPosition', ['column', 'line'])
15
16 # In Python 3 we'd use __prepare__ and an ordered dict...
17 _TOKREGISTER = []
18
19
20 class _LexerMeta(type):
21 def __new__(mcs, name, bases, attrs):
22 global _TOKREGISTER # pylint: disable=W0603
23 try:
24 attrs['__tokens__'] = (set(), []) # Set of token names, list of (rx, callback, defaultType)
25 klass = super().__new__(mcs, name, bases, attrs)
26 for func, rx, toktypes in _TOKREGISTER:
27 klass.add_token_type(func.__name__, callback_by_name(func.__name__), rx, toktypes)
28 return klass
29 finally:
30 _TOKREGISTER = []
31
32
33 def token(rx, types=None):
34 """
35 The method decorator for tokens
36 """
37 def _wrap(func):
38 if any(func.__name__ == aFunc.__name__ and func != aFunc for aFunc, _, _ in _TOKREGISTER):
39 raise TypeError(f'Duplicate token method name "{func.__name__}"')
40 _TOKREGISTER.append((func, rx, types))
41 return func
42 return _wrap
43
44
45 class SkipToken(Exception):
46 """
47 Raise this from your consumer to ignore the token.
48 """
49
50
51 class LexerError(Exception):
52 """
53 Unrecognized token in input
54
55 :ivar lineno: Line in input
56 :ivar colno: Column in input
57 """
58 def __init__(self, char, pos):
59 super().__init__(f'Unrecognized token {repr(char)}')
60 self.position = pos
61
62 # Getters for compatibility with <1.3.8
63 @property
64 def colno(self):
65 """
66 Column
67 """
68 return self.position.column
69 @property
70 def lineno(self):
71 """
72 Line
73 """
74 return self.position.line
75
76
77 class EOF(metaclass=Singleton):
78 """
79 End symbol
80 """
81
82 __reprval__ = '$'
83
84 @property
85 def type(self):
86 """Read-only attribute for Token duck-typing"""
87 return self
88 @property
89 def value(self):
90 """Read-only attribute for Token duck-typing"""
91 return self
92
93
94 class LexerBase(metaclass=_LexerMeta):
95 """
96 This defines the interface for lexer classes. For concrete
97 implementations, see :py:class:`ProgressiveLexer` and
98 :py:class:`ReLexer`.
99 """
100
101 Token = collections.namedtuple('Token', ['type', 'value', 'position'])
102
103 # Shut up pychecker. Those are actually set by the metaclass.
104 __tokens__ = ()
105
106 class _MutableToken: # pylint: disable=too-few-public-methods
107 def __init__(self, type_, value, position):
108 self.type = type_
109 self.value = value
110 self.position = position
111
112 def token(self):
113 """Returns the unmutable equivalent"""
114 return EOF if EOF in [self.type, self.value] else LexerBase.Token(self.type, self.value, self.position)
115
116 def __init__(self):
117 super().__init__()
118
119 self._pos = None
120 self._consumer = None
121
122 self.restart_lexer()
123
124 def restart_lexer(self, reset_pos=True):
125 if reset_pos:
126 self._pos = LexerPosition(column=1, line=1)
127 self._input = []
128 self._consumer = None
129
130 def restartLexer(self, resetPos=True): # pylint: disable=invalid-name
131 warnings.warn('restartLexer is deprecated in favor of restart_lexer', DeprecationWarning)
132 self.restart_lexer(reset_pos=resetPos)
133
134 def position(self):
135 """
136 :return: The current position in stream as a 2-tuple (column, line).
137 """
138 return self._pos
139
140 def advance_column(self, count=1):
141 """
142 Advances the current position by *count* columns.
143 """
144 self._pos = self._pos._replace(column=self._pos.column + count)
145
146 def advanceColumn(self, count=1): # pylint: disable=invalid-name
147 warnings.warn('advanceColumn is deprecated in favor of advance_column', DeprecationWarning)
148 self.advance_column(count=count)
149
150 def advance_line(self, count=1):
151 """
152 Advances the current position by *count* lines.
153 """
154 self._pos = self._pos._replace(column=1, line=self._pos.line + count)
155
156 def advanceLine(self, count=1): # pylint: disable=invalid-name
157 warnings.warn('advanceLine is deprecated in favor of advance_line', DeprecationWarning)
158 self.advance_line(count=count)
159
160 @staticmethod
161 def ignore(char):
162 """
163 Override this to ignore characters in input stream. The
164 default is to ignore spaces and tabs.
165
166 :param char: The character to test
167 :return: True if *char* should be ignored
168 """
169 return char in chars(' ') + chars('\t')
170
171 def set_consumer(self, consumer):
172 """
173 Sets the current consumer. A consumer is an object with a
174 *feed* method; all characters seen on the input stream after
175 the consumer is set are passed directly to it. When the *feed*
176 method returns a 2-tuple (type, value), the corresponding
177 token is generated and the consumer reset to None. This may be
178 handy to parse tokens that are not easily recognized by a
179 regular expression but easily by code; for instance the
180 following lexer recognizes C strings without having to use
181 negative lookahead:
182
183 .. code-block:: python
184
185 class MyLexer(ReLexer):
186 @token('"')
187 def cstring(self, tok):
188 class CString:
189 def __init__(self):
190 self.state = 0
191 self.value = StringIO.StringIO()
192 def feed(self, char):
193 if self.state == 0:
194 if char == '"':
195 return 'cstring', self.value.getvalue()
196 if char == '\\\\':
197 self.state = 1
198 else:
199 self.value.write(char)
200 elif self.state == 1:
201 self.value.write(char)
202 self.state = 0
203 self.set_consumer(CString())
204
205 You can also raise SkipToken instead of returning a token if it
206 is to be ignored (comments).
207 """
208 self._consumer = consumer
209
210 def setConsumer(self, consumer): # pylint: disable=invalid-name
211 warnings.warn('setConsumer is deprecated in favor of set_consumer', DeprecationWarning)
212 self.set_consumer(consumer)
213
214 def consumer(self):
215 return self._consumer
216
217 def parse(self, string): # pragma: no cover
218 """
219 Parses the whole *string*; returns the start symbol semantic value
220 """
221 raise NotImplementedError
222
223 def new_token(self, tok): # pragma: no cover
224 """
225 This method will be invoked as soon as a token is recognized on input.
226
227 :param tok: The token. This is a named tuple with *type* and *value* attributes.
228 """
229 raise NotImplementedError
230
231 def newToken(self, tok): # pylint: disable=invalid-name
232 warnings.warn('newToken is deprecated in favor of new_token', DeprecationWarning)
233 self.new_token(tok)
234
235 @classmethod
236 def add_token_type(cls, name, callback, regex, types=None):
237 for type_name in [name] if types is None else types:
238 if type_name is not EOF:
239 cls.__tokens__[0].add(type_name)
240 cls.__tokens__[1].append((regex, callback, name if types is None else None))
241
242 @classmethod
243 def _all_tokens(cls):
244 tokens = (set(), [])
245 for base in inspect.getmro(cls):
246 if issubclass(base, LexerBase):
247 tokens[0].update(base.__tokens__[0])
248 tokens[1].extend(base.__tokens__[1])
249 return tokens
250
251 @classmethod
252 def token_types(cls):
253 """
254 :return: the set of all token names, as strings.
255 """
256 return cls._all_tokens()[0]
257
258 @classmethod
259 def tokenTypes(cls): # pylint: disable=invalid-name
260 warnings.warn('tokenTypes is deprecated in favor of token_types', DeprecationWarning)
261 return cls.token_types()
262
263
264 class ReLexer(LexerBase): # pylint: disable=W0223
265 """
266 Concrete lexer based on Python regular expressions.
267 """
268 def __init__(self):
269 self._regexes = []
270 for rx, callback, default_type in self._all_tokens()[1]:
271 crx = re.compile((b'^' if isinstance(rx, bytes) else '^') + rx)
272 self._regexes.append((crx, callback, default_type))
273 super().__init__()
274
275 def _parse(self, string, pos):
276 # pylint: disable=too-many-nested-blocks
277 while pos < len(string):
278 char = string[pos]
279 try:
280 if self.consumer() is None:
281 if self.ignore(char):
282 pos += 1
283 continue
284 pos = self._find_match(string, pos)
285 else:
286 try:
287 tok = self.consumer().feed(char)
288 except SkipToken:
289 self.set_consumer(None)
290 else:
291 if tok is not None:
292 self.set_consumer(None)
293 if tok[0] is not None:
294 self.new_token(self.Token(*tok, self.position()))
295 pos += 1
296 finally:
297 if char in chars('\n'):
298 self.advance_line()
299 else:
300 self.advance_column()
301 return pos
302
303 def parse(self, string):
304 try:
305 self._parse(string, 0)
306 return self.new_token(EOF)
307 except LexerError:
308 self.restart_lexer()
309 raise
310
311 def _find_match(self, string, pos):
312 match = None
313 matchlen = 0
314 pos2d = self.position()
315
316 for rx, callback, default_type in self._regexes:
317 mtc = rx.match(string[pos:])
318 if mtc:
319 value = mtc.group(0)
320 if len(value) > matchlen:
321 match = value, callback, default_type
322 matchlen = len(value)
323
324 if match:
325 value, callback, default_type = match
326 tok = self._MutableToken(default_type, value, pos2d)
327 callback(self, tok)
328 pos += matchlen
329 if self.consumer() is None and tok.type is not None:
330 self.new_token(tok.token())
331 self.advance_column(matchlen - 1)
332 return pos
333
334 raise LexerError(self._guess_token(string, pos), pos2d)
335
336 def _guess_token(self, string, pos):
337 start = pos
338 while True:
339 pos += 1
340 if pos == len(string) or self.ignore(string[pos]):
341 break
342
343 for rx, _, _ in self._regexes:
344 mtc = rx.match(string[pos:])
345 if mtc:
346 break
347 else:
348 continue
349 break
350 return string[start:pos]