Debian Codegraph @

ptk/parser.py

raw
1 # -*- coding: utf-8 -*-
2
3 # (c) Jérôme Laheurte 2015-2026
4 # See LICENSE.txt
5
6 import functools
7 import collections
8 import logging
9 import re
10 import warnings
11
12 from ptk.lexer import EOF, token, LexerPosition, ReLexer
13 from ptk.grammar import Grammar, Production, GrammarError
14 # production is only imported so that client code doesn't have to import it from grammar
15 from ptk.grammar import production # pylint: disable=W0611
16 from ptk.utils import Singleton, callback_by_name
17
18
19 class ParseError(Exception):
20 """
21 Syntax error when parsing.
22
23 :ivar token: The unexpected token.
24 """
25 def __init__(self, grammar, tok, state, tokens):
26 self.token = tok
27 super().__init__(f'Unexpected token "{tok.value}" ({tok.type}) in state "{sorted(state)}"')
28
29 self._state = state
30 self._expecting = set()
31 for terminal in grammar.token_types():
32 if grammar.__actions__.get((state, terminal), None) is not None:
33 self._expecting.add(terminal)
34 self._tokens = tokens
35
36 @property
37 def position(self):
38 return self.token.position
39
40 def expecting(self):
41 """
42 Returns a set of tokens types that would have been valid in input.
43 """
44 return self._expecting
45
46 def state(self):
47 """
48 Returns the parser state when the error was encountered.
49 """
50 return self._state
51
52 def last_token(self):
53 """
54 Returns the last valid token seen before this error
55 """
56 return self._tokens[-1]
57
58 def lastToken(self): # pylint: disable=invalid-name
59 warnings.warn('lastToken is deprecated in favor of last_token', DeprecationWarning)
60 return self.last_token()
61
62 def tokens(self):
63 """
64 Returns all tokens seen
65 """
66 return self._tokens
67
68
69 def left_assoc(*operators):
70 """
71 Class decorator for left associative operators. Use this to
72 decorate your :py:class:`Parser` class. Operators passed as
73 argument are assumed to have the same priority. The later you
74 declare associativity, the higher the priority; so the following
75 code
76
77 .. code-block:: python
78
79 @left_assoc('+', '-')
80 @left_assoc('*', '/')
81 class MyParser(LRParser):
82 # ...
83
84 declares '+' and '-' to be left associative, with the same
85 priority. '*' and '/' are also left associative, with a higher
86 priority than '+' and '-'.
87
88 See also the *priority* argument to :py:func:`production`.
89 """
90 def _wrapper(cls):
91 cls.__precedence__.insert(0, ('left', set(operators)))
92 return cls
93 return _wrapper
94
95
96 def leftAssoc(*operators): # pylint: disable=invalid-name
97 warnings.warn('leftAssoc is deprecated in favor of left_assoc', DeprecationWarning)
98 return left_assoc(*operators)
99
100
101 def right_assoc(*operators):
102 """
103 Class decorator for right associative operators. Same remarks as :py:func:`left_assoc`.
104 """
105 def _wrapper(cls):
106 cls.__precedence__.insert(0, ('right', set(operators)))
107 return cls
108 return _wrapper
109
110
111 def rightAssoc(*operators): # pylint: disable=invalid-name
112 warnings.warn('rightAssoc is deprecated in favor of right_assoc', DeprecationWarning)
113 return right_assoc(*operators)
114
115
116 def non_assoc(*operators):
117 """
118 Class decorator for non associative operators. Same remarks as :py:func:`left_assoc`.
119 """
120 def _wrapper(cls):
121 cls.__precedence__.insert(0, ('non', set(operators)))
122 return cls
123 return _wrapper
124
125
126 def nonAssoc(*operators): # pylint: disable=invalid-name
127 warnings.warn('nonAssoc is deprecated in favor of non_assoc', DeprecationWarning)
128 return non_assoc(*operators)
129
130
131 class _StartState(metaclass=Singleton): # pylint: disable=too-few-public-methods
132 __reprval__ = '\u03A3'
133
134
135 class _ResolveError(Exception):
136 pass
137
138
139 @functools.total_ordering
140 class _Item:
141 __slots__ = ('production', 'dot', 'terminal', 'index', 'should_reduce', 'expecting')
142
143 def __init__(self, prod, dot, terminal):
144 self.production = prod
145 self.dot = dot
146 self.terminal = terminal
147 self.index = None
148 self.should_reduce = self.dot == len(self.production.right)
149 self.expecting = None if self.should_reduce else self.production.right[self.dot]
150
151 def next(self):
152 """
153 Returns an item with the dot advanced one position
154 """
155 return _Item(self.production, self.dot + 1, self.terminal)
156
157 def __repr__(self):
158 symbols = list(self.production.right)
159 symbols.insert(self.dot, '\u2022')
160 return '%s -> %s (%s)' % (self.production.name, ' '.join([repr(sym) for sym in symbols]), self.terminal) # pylint: disable=consider-using-f-string
161
162 def __eq__(self, other):
163 return (self.production, self.dot, self.terminal) == (other.production, other.dot, other.terminal)
164
165 def __lt__(self, other):
166 return (self.production, self.dot, self.terminal) < (other.production, other.dot, other.terminal)
167
168 def __hash__(self):
169 return hash((self.production, self.dot, self.terminal))
170
171
172 class _Accept(BaseException):
173 def __init__(self, result):
174 self.result = result
175 super().__init__()
176
177
178 _StackItem = collections.namedtuple('_StackItem', ['state', 'value', 'position'])
179
180
181 class _Shift: # pylint: disable=too-few-public-methods
182 def __init__(self, new_state):
183 self.new_state = new_state
184
185 def do_action(self, grammar, stack, tok): # pylint: disable=W0613
186 stack.append(_StackItem(self.new_state, tok.value, tok.position))
187 return True
188
189
190 class _Reduce: # pylint: disable=too-few-public-methods
191 def __init__(self, item):
192 self.item = item
193 self.nargs = len(item.production.right)
194
195 def do_action(self, grammar, stack, tok): # pylint: disable=W0613
196 pos, (callback, kwargs) = self._get_callback(stack)
197 self._applied(grammar, stack, callback(grammar, **kwargs), pos)
198 return False
199
200 def _applied(self, grammar, stack, prod_val, position):
201 stack.append(_StackItem(grammar.goto(stack[-1].state, self.item.production.name), prod_val, position))
202
203 def _get_callback(self, stack):
204 if self.nargs:
205 args = [stackItem.value for stackItem in stack[-self.nargs:]]
206 pos = stack[-self.nargs].position
207 stack[-self.nargs:] = []
208 else:
209 args = []
210 pos = stack[-1].position # Hum.
211 return pos, self.item.production.apply(args, pos)
212
213
214 class LRParser(Grammar):
215 """
216 LR(1) parser. This class is intended to be used with a lexer class
217 derived from :py:class:`LexerBase`, using inheritance; it
218 overrides :py:func:`LexerBase.new_token` so you must inherit from
219 the parser first, then the lexer:
220
221 .. code-block:: python
222
223 class MyParser(LRParser, ReLexer):
224 # ...
225
226 """
227 def __init__(self): # pylint: disable=R0914,R0912
228 super().__init__()
229 self._restart_parser()
230
231 def rstack(self):
232 return reversed(self.__stack)
233
234 def new_token(self, tok):
235 try:
236 for action, stack in self._process_token(tok):
237 if action.do_action(self, stack, tok):
238 break
239 self.__tokens.append(tok)
240 return None
241 except _Accept as exc:
242 self._restart_parser()
243 self.on_start_symbol(exc.result)
244 return exc.result
245
246 def _process_token(self, tok):
247 while True:
248 action = self.__actions__.get((self.__stack[-1].state, tok.type), None)
249 if action is None:
250 raise ParseError(self, tok, self.__stack[-1].state, self.__tokens)
251 yield action, self.__stack
252
253 def on_start_symbol(self, start_symbol): # pragma: no cover
254 """
255 This is called when the start symbol has been reduced.
256
257 :param start_symbol: The value associated with the start symbol.
258 """
259 return self.newSentence(start_symbol)
260
261 def newSentence(self, start_symbol): # pylint: disable=invalid-name,unused-argument
262 warnings.warn('newSentence has been deprecated in favor of on_start_symbol', DeprecationWarning)
263 raise NotImplementedError
264
265 @classmethod
266 def _create_production_parser(cls, name, priority, attrs):
267 return ProductionParser(callback_by_name(name), priority, cls, attrs)
268
269 @classmethod
270 def _create_reduce_action(cls, item):
271 return _Reduce(item)
272
273 @classmethod
274 def _create_shift_action(cls, state):
275 return _Shift(state)
276
277 @classmethod
278 def prepare(cls):
279 for prod in cls.productions():
280 if prod.name is _StartState:
281 break
282 else:
283 def acceptor(_, result):
284 raise _Accept(result)
285 prod = Production(_StartState, acceptor)
286 prod.add_symbol(cls._default_start_symbol() if cls.startSymbol is None else cls.startSymbol, name='result')
287 cls.__productions__.insert(0, prod)
288
289 cls.startSymbol = _StartState
290 super().prepare()
291
292 states, goto = cls.__compute_states(prod)
293 reachable = cls.__compute_actions(states, goto)
294
295 logger = logging.getLogger('LRParser')
296 cls.__resolve_conflicts(logger)
297
298 used_tokens = {symbol for _, symbol in cls.__actions__ if symbol is not EOF}
299 if used_tokens != cls.token_types(): # pragma: no cover pylint: disable=no-member
300 logger.warning('The following tokens are not used: %s', ','.join([repr(sym) for sym in sorted(cls.token_types() - used_tokens)])) # pylint: disable=no-member
301
302 if reachable != cls.nonterminals(): # pragma: no cover
303 logger.warning('The following nonterminals are not reachable: %s', ','.join([repr(sym) for sym in sorted(cls.nonterminals() - reachable)]))
304
305 # Reductions only need goto entries for nonterminals
306 cls._goto = {(state, symbol): new_state for (state, symbol), new_state in goto.items() if symbol not in cls.token_types()} # pylint: disable=no-member
307
308 parts = []
309 if cls.nSR:
310 parts.append(f'{cls.nSR} shift/reduce conflicts')
311 if cls.nRR:
312 parts.append(f'{cls.nRR} reduce/reduce conflicts')
313 if parts:
314 logger.warning(', '.join(parts))
315
316 # Cast to tuple because sets are not totally ordered
317 for index, state in enumerate([tuple(cls._startState)] + sorted([tuple(state) for state in states if state != cls._startState])):
318 logger.debug('State %d', index)
319 for item in sorted(state):
320 logger.debug(' %s', item)
321 item.index = index
322 cls.__lrstates__.append(sorted(state)) # pylint: disable=no-member
323 logger.info('%d states.', len(states))
324
325 @classmethod
326 def __compute_states(cls, start):
327 all_syms = list(cls.token_types() | cls.nonterminals()) # pylint: disable=no-member
328 goto = []
329 cls._startState = frozenset([_Item(start, 0, EOF)])
330 states = set([cls._startState])
331 stack = [cls._startState]
332 while stack:
333 state = stack.pop()
334 state_closure = cls.__item_set_closure(state)
335 for symbol in all_syms:
336 # Compute goto(symbol, state)
337 next_state = frozenset([item.next() for item in state_closure if item.expecting == symbol])
338 if next_state:
339 goto.append(((state, symbol), next_state))
340 if next_state not in states:
341 states.add(next_state)
342 stack.append(next_state)
343 return states, dict(goto)
344
345 @classmethod
346 def __compute_actions(cls, states, goto):
347 cls.__actions__ = {}
348 reachable = set()
349 token_types = cls.token_types() # pylint: disable=no-member
350 for state in states:
351 for item in cls.__item_set_closure(state):
352 if item.should_reduce:
353 action = cls._create_reduce_action(item)
354 reachable.add(item.production.name)
355 cls.__add_reduce_action(state, item.terminal, action)
356 else:
357 symbol = item.production.right[item.dot]
358 if symbol in token_types:
359 cls.__add_shift_action(state, symbol, cls._create_shift_action(goto[(state, symbol)]))
360 return reachable
361
362 @classmethod
363 def __should_prefer_shift(cls, logger, reduce_action, symbol):
364 logger.info('Shift/reduce conflict for "%s" on "%s"', reduce_action.item, symbol)
365
366 prod_precedence = reduce_action.item.production.precedence(cls)
367 token_precedence = cls.terminal_precedence(symbol)
368
369 # If both precedences are specified, use priority/associativity
370 if prod_precedence is not None and token_precedence is not None:
371 prod_assoc, prod_prio = prod_precedence
372 token_assoc, token_prio = token_precedence
373 if prod_prio > token_prio:
374 logger.info('Resolving in favor of reduction because of priority')
375 return False
376 if prod_prio < token_prio:
377 logger.info('Resolving in favor of shift because of priority')
378 return True
379 if prod_assoc == token_assoc:
380 if prod_assoc == 'non':
381 logger.info('Resolving in favor of error because of associativity')
382 raise _ResolveError()
383 if prod_assoc == 'left':
384 logger.info('Resolving in favor of reduction because of associativity')
385 return False
386 logger.info('Resolving in favor of shift because of associativity')
387 return True
388
389 # At least one of those is not specified; use shift
390 logger.warning('Could not disambiguate shift/reduce conflict for "%s" on "%s"; using shift', reduce_action.item, symbol)
391 cls.nSR += 1
392 return True
393
394 @classmethod
395 def __resolve_conflicts(cls, logger):
396 cls.nSR = 0
397 cls.nRR = 0
398
399 for (state, symbol), actions in sorted(cls.__actions__.items()):
400 action = actions.pop()
401 while actions:
402 conflicting = actions.pop()
403 try:
404 action = cls.__resolve_conflict(logger, action, conflicting, symbol)
405 except _ResolveError:
406 del cls.__actions__[(state, symbol)]
407 break
408 else:
409 cls.__actions__[(state, symbol)] = action
410
411 @classmethod
412 def __resolve_conflict(cls, logger, action1, action2, symbol):
413 if isinstance(action2, _Shift):
414 action1, action2 = action2, action1
415
416 if isinstance(action1, _Shift):
417 # Shift/reduce
418 return action1 if cls.__should_prefer_shift(logger, action2, symbol) else action2
419
420 # Reduce/reduce
421 logger.warning('Reduce/reduce conflict between "%s" and "%s"', action1.item, action2.item)
422 cls.nRR += 1
423
424 # Use the first one to be declared
425 for prod in cls.productions():
426 if prod == action1.item.production:
427 logger.warning('Using "%s', prod)
428 return action1
429 if prod == action2.item.production:
430 logger.warning('Using "%s', prod)
431 return action2
432
433 return None
434
435 @classmethod
436 def __add_reduce_action(cls, state, symbol, action):
437 cls.__actions__.setdefault((state, symbol), []).append(action)
438
439 @classmethod
440 def __add_shift_action(cls, state, symbol, action):
441 for existing in cls.__actions__.get((state, symbol), []):
442 if isinstance(existing, _Shift):
443 return
444 cls.__actions__.setdefault((state, symbol), []).append(action)
445
446 @classmethod
447 def goto(cls, state, symbol):
448 return cls._goto[(state, symbol)]
449
450 def _restart_parser(self):
451 self.__stack = [_StackItem(self._startState, None, LexerPosition(1, 1))]
452 self.__tokens = []
453 self.restart_lexer() # pylint: disable=no-member
454
455 @classmethod
456 @functools.cache
457 def __item_set_closure(cls, items):
458 result = set(items)
459 while True:
460 prev = set(result)
461 for item in [item for item in result if not item.should_reduce]:
462 symbol = item.production.right[item.dot]
463 if symbol not in cls.token_types(): # pylint: disable=no-member
464 terminals = cls.first(*tuple(item.production.right[item.dot + 1:] + [item.terminal]))
465 for prod in (prod for prod in cls.productions() if prod.name == symbol):
466 for terminal in terminals:
467 result.add(_Item(prod, 0, terminal))
468 if prev == result:
469 break
470 return result
471
472
473 class ProductionParser(LRParser, ReLexer): # pylint: disable=R0904
474 """
475 Parser for productions in other parsers
476 """
477 def __init__(self, callback, priority, grammar_class, attributes): # pylint: disable=R0915
478 self.callback = callback
479 self.priority = priority
480 self.grammar_class = grammar_class
481 self.attributes = attributes
482
483 super().__init__()
484
485 @classmethod
486 def prepare(cls, **kwargs): # pylint: disable=R0915
487 # Obviously cannot use @production here
488
489 # When mixing async and sync parsers in the same program this may be called twice,
490 # because AsyncProductionParser inherits from ProductionParser
491 if cls.productions():
492 # FIXME remove this, we don't do the async stuff any more
493 return
494
495 # DECL -> identifier "->" PRODS
496 prod = Production('DECL', cls.DECL)
497 prod.add_symbol('LEFT', 'left')
498 prod.add_symbol('arrow')
499 prod.add_symbol('PRODS', 'prods')
500 cls.__productions__.append(prod)
501
502 # LEFT -> identifier
503 prod = Production('LEFT', cls.LEFT)
504 prod.add_symbol('identifier', 'name')
505 cls.__productions__.append(prod)
506
507 # LEFT -> identifier "<" posarg ">"
508 prod = Production('LEFT', cls.LEFT)
509 prod.add_symbol('identifier', 'name')
510 prod.add_symbol('lchev')
511 prod.add_symbol('identifier', 'posarg')
512 prod.add_symbol('rchev')
513 cls.__productions__.append(prod)
514
515 # PRODS -> P
516 prod = Production('PRODS', cls.PRODS1)
517 prod.add_symbol('P', 'prodlist')
518 cls.__productions__.append(prod)
519
520 # PRODS -> PRODS "|" P
521 prod = Production('PRODS', cls.PRODS2)
522 prod.add_symbol('PRODS', 'prods')
523 prod.add_symbol('union')
524 prod.add_symbol('P', 'prodlist')
525 cls.__productions__.append(prod)
526
527 # P -> P SYM
528 prod = Production('P', cls.P1)
529 prod.add_symbol('P', 'prodlist')
530 prod.add_symbol('SYM', 'sym')
531 cls.__productions__.append(prod)
532
533 # P -> ɛ
534 prod = Production('P', cls.P2)
535 cls.__productions__.append(prod)
536
537 # SYM -> SYMNAME PROPERTIES
538 prod = Production('SYM', cls.SYM)
539 prod.add_symbol('SYMNAME', 'symname')
540 prod.add_symbol('PROPERTIES', 'properties')
541 cls.__productions__.append(prod)
542
543 # SYM -> SYMNAME repeat PROPERTIES
544 prod = Production('SYM', cls.SYMREP)
545 prod.add_symbol('SYMNAME', 'symname')
546 prod.add_symbol('repeat', 'repeat')
547 prod.add_symbol('PROPERTIES', 'properties')
548 cls.__productions__.append(prod)
549
550 # SYM -> SYMNAME repeat lparen identifier rparen PROPERTIES
551 prod = Production('SYM', cls.SYMREP)
552 prod.add_symbol('SYMNAME', 'symname')
553 prod.add_symbol('repeat', 'repeat')
554 prod.add_symbol('lparen')
555 prod.add_symbol('identifier', 'separator')
556 prod.add_symbol('rparen')
557 prod.add_symbol('PROPERTIES', 'properties')
558 cls.__productions__.append(prod)
559
560 # SYM -> SYMNAME repeat lparen litteral rparen PROPERTIES
561 prod = Production('SYM', cls.SYMREP_LIT)
562 prod.add_symbol('SYMNAME', 'symname')
563 prod.add_symbol('repeat', 'repeat')
564 prod.add_symbol('lparen')
565 prod.add_symbol('litteral', 'separator')
566 prod.add_symbol('rparen')
567 prod.add_symbol('PROPERTIES', 'properties')
568 cls.__productions__.append(prod)
569
570 # SYMNAME -> identifier
571 prod = Production('SYMNAME', cls.SYMNAME1)
572 prod.add_symbol('identifier', 'identifier')
573 cls.__productions__.append(prod)
574
575 # SYMNAME -> litteral
576 prod = Production('SYMNAME', cls.SYMNAME2)
577 prod.add_symbol('litteral', 'litteral')
578 cls.__productions__.append(prod)
579
580 # PROPERTIES -> ɛ
581 prod = Production('PROPERTIES', cls.PROPERTIES1)
582 cls.__productions__.append(prod)
583
584 # PROPERTIES -> lchev identifier rchev
585 prod = Production('PROPERTIES', cls.PROPERTIES2)
586 prod.add_symbol('lchev')
587 prod.add_symbol('identifier', 'name')
588 prod.add_symbol('rchev')
589 cls.__productions__.append(prod)
590
591 super().prepare(**kwargs)
592
593 def on_start_symbol(self, start_symbol):
594 (name, posarg), prods = start_symbol
595 for prod in prods:
596 if prod.name is None:
597 prod.name = name
598 prod.posarg = posarg
599 self.grammar_class.__productions__.extend(prods)
600
601 # Lexer
602
603 @staticmethod
604 def ignore(char):
605 return char in ' \t\n'
606
607 @token('->')
608 def arrow(self, tok):
609 pass
610
611 @token('<')
612 def lchev(self, tok):
613 pass
614
615 @token('>')
616 def rchev(self, tok):
617 pass
618
619 @token(r'\|')
620 def union(self, tok):
621 pass
622
623 @token(r'\*|\+|\?')
624 def repeat(self, tok):
625 pass
626
627 @token(r'\(')
628 def lparen(self, tok):
629 pass
630
631 @token(r'\)')
632 def rparen(self, tok):
633 pass
634
635 @token('[a-zA-Z_][a-zA-Z0-9_]*')
636 def identifier(self, tok):
637 pass
638
639 @token(r'"|\'')
640 def litteral(self, tok):
641 class StringBuilder: # pylint: disable=missing-class-docstring,too-few-public-methods
642 def __init__(self, quotetype):
643 self.quotetype = quotetype
644 self.chars = []
645 self.state = 0
646 def feed(self, char):
647 if self.state == 0:
648 if char == '\\':
649 self.state = 1
650 elif char == self.quotetype:
651 return 'litteral', ''.join(self.chars)
652 else:
653 self.chars.append(char)
654 elif self.state == 1:
655 self.chars.append(char)
656 self.state = 0
657 return None
658 self.set_consumer(StringBuilder(tok.value))
659
660 # Parser
661
662 def DECL(self, left, prods): # pylint: disable=invalid-name
663 name, _ = left
664 if name in self.grammar_class.token_types():
665 raise GrammarError(f'"{name}" is a token name and cannot be used as non-terminal')
666 return (left, prods)
667
668 def LEFT(self, name, posarg=None): # pylint: disable=invalid-name
669 return (name, posarg)
670
671 def PRODS1(self, prodlist): # pylint: disable=invalid-name
672 return prodlist
673
674 def PRODS2(self, prods, prodlist): # pylint: disable=invalid-name
675 prods.extend(prodlist)
676 return prods
677
678 def P1(self, sym, prodlist): # pylint: disable=invalid-name
679 result = []
680 symbol, properties, repeat, sep = sym
681
682 for prod in prodlist:
683 if prod.name is None:
684 if repeat is None:
685 prod.add_symbol(symbol, name=properties.get('name', None))
686 result.append(prod)
687 elif repeat == '?':
688 if sep is not None:
689 raise GrammarError('A separator makes no sense for "?"')
690 self.__add_at_most_one(result, prod, symbol, properties.get('name', None))
691 elif repeat in ['*', '+']:
692 self.__add_list(result, prod, symbol, properties.get('name', None), repeat == '*', sep)
693 else:
694 result.append(prod)
695
696 return result
697
698 def __add_at_most_one(self, productions, prod, symbol, name):
699 clone = prod.cloned()
700 if name is not None:
701 self._wrap_callback_none(name, clone)
702 productions.append(clone)
703
704 prod.add_symbol(symbol, name=name)
705 productions.append(prod)
706
707 def _wrap_callback_none(self, name, prod):
708 previous = prod.callback
709 def callback(*args, **kwargs):
710 kwargs[name] = None
711 return previous(*args, **kwargs)
712 prod.callback = callback
713
714 def __add_list(self, productions, prod, symbol, name, allow_empty, sep): # pylint: disable=too-many-arguments,too-many-positional-arguments
715 class ListSymbol(metaclass=Singleton): # pylint: disable=missing-class-docstring,too-few-public-methods
716 __reprval__ = 'List(%s, "%s")' % (symbol, '*' if allow_empty else '+') # pylint: disable=consider-using-f-string
717
718 if allow_empty:
719 clone = prod.cloned()
720 self._wrap_callback_empty(name, clone)
721 productions.append(clone)
722
723 prod.add_symbol(ListSymbol, name=name)
724 productions.append(prod)
725
726 list_prod = Production(ListSymbol, self._wrap_callback_one())
727 list_prod.add_symbol(symbol, name='item')
728 productions.append(list_prod)
729
730 list_prod = Production(ListSymbol, self._wrap_callback_next())
731 list_prod.add_symbol(ListSymbol, name='items')
732 if sep is not None:
733 list_prod.add_symbol(sep)
734 list_prod.add_symbol(symbol, name='item')
735 productions.append(list_prod)
736
737 def _wrap_callback_empty(self, name, prod):
738 previous = prod.callback
739 def callback_empty(*args, **kwargs):
740 if name is not None:
741 kwargs[name] = []
742 return previous(*args, **kwargs)
743 prod.callback = callback_empty
744
745 def _wrap_callback_one(self):
746 def callback_one(_, item):
747 return [item]
748 return callback_one
749
750 def _wrap_callback_next(self):
751 def callback_next(_, items, item):
752 items.append(item)
753 return items
754 return callback_next
755
756 def P2(self): # pylint: disable=invalid-name
757 # 'name' is replaced in on_start_symbol()
758 return [Production(None, self.callback, priority=self.priority, attributes=self.attributes)]
759
760 def SYMNAME1(self, identifier): # pylint: disable=invalid-name
761 return identifier
762
763 def SYMNAME2(self, litteral): # pylint: disable=invalid-name
764 name = litteral
765 if name not in self.grammar_class.token_types():
766 self.grammar_class.add_token_type(name, lambda s, tok: None, re.escape(name), None)
767 return name
768
769 def SYM(self, symname, properties): # pylint: disable=invalid-name
770 return (symname, properties, None, None)
771
772 def SYMREP(self, symname, repeat, properties, separator=None): # pylint: disable=invalid-name
773 return (symname, properties, repeat, separator)
774
775 def SYMREP_LIT(self, symname, repeat, properties, separator): # pylint: disable=invalid-name
776 if separator not in self.grammar_class.token_types():
777 self.grammar_class.add_token_type(separator, lambda s, tok: None, re.escape(separator), None)
778 return self.SYMREP(symname, repeat, properties, separator)
779
780 def PROPERTIES1(self): # pylint: disable=invalid-name
781 return {}
782
783 def PROPERTIES2(self, name): # pylint: disable=invalid-name
784 return {'name': name}