Debian Codegraph @

samples/yacc2py.py

raw
1 #!/usr/bin/env python3
2 # -*- coding: utf-8 -*-
3
4 """
5 Converts a Yacc/Bison grammar definition into a Python skeleton that uses ptk.
6 """
7
8 import io
9 import getopt
10 import sys
11 import collections
12 import codecs
13 import re
14
15 from ptk.parser import production, LRParser, ParseError
16 from ptk.lexer import token, ReLexer, EOF
17 from ptk.regex import buildRegex, DeadState
18
19
20 Symbol = collections.namedtuple('Symbol', ('name', 'argname'))
21
22
23 class Options(object):
24 def __init__(self, opts):
25 self.compact = False
26 self.arguments = False
27 self.filename = None
28 for opt, val in opts:
29 if opt in ('-c', '--compact'):
30 self.compact = True
31 if opt in ('-a', '--arguments'):
32 self.arguments = True
33 if opt in ('-o', '--output'):
34 self.filename = val
35 if opt in ('-h', '--help'):
36 self.usage()
37
38 if self.compact and self.arguments:
39 print('--compact and --arguments are not compatible')
40 self.usage(1)
41
42 if self.filename is None:
43 print('Output file not specified')
44 self.usage(1)
45
46 def usage(self, exitCode=0):
47 print('Usage: %s [options] filename' % sys.argv[0])
48 print('Options:')
49 print(' -h, --help Print this')
50 print(' -c, --compact Create one method for all alternatives of a production')
51 print(' -o, --output <filename> Output to file (mandatory)')
52 print(' -a, --arguments Generate argument names for items in productions (incompatible with --compact)')
53 sys.exit(exitCode)
54
55 @staticmethod
56 def create():
57 opts, args = getopt.getopt(sys.argv[1:], 'caho:', ['compact', 'arguments', 'help', 'output='])
58 return Options(opts), args
59
60
61 class NullToken(object):
62 def __init__(self, endMarker):
63 self.__rx = buildRegex('(.|\n)*%s' % re.escape(endMarker)).start()
64
65 def feed(self, char):
66 try:
67 if self.__rx.feed(char):
68 return None, None
69 except DeadState:
70 return None, None
71
72
73 class YaccParser(LRParser, ReLexer):
74 def __init__(self, options, stream):
75 self.stream = stream
76 self.options = options
77 super().__init__()
78
79 self.state = 0
80 self.yaccStartSymbol = None
81 self.allTokens = list()
82 self.allProductions = list()
83 self.precedences = list()
84
85 # Lexer
86
87 @token(r'%\{', types=[])
88 def c_decl(self, tok):
89 self.set_consumer(NullToken('%}'))
90
91 @token(r'/\*', types=[])
92 def comment(self, tok):
93 self.set_consumer(NullToken('*/'))
94
95 @token(r'%union\s*{', types=[]) # Hum, no LF possible before {
96 def union(self, tok):
97 self.set_consumer(NullToken('}'))
98
99 @token(r'%%')
100 def part_sep(self, tok):
101 self.state += 1
102 if self.state == 2:
103 # Ignore C code after last %%
104 class IgnoreCCode(object):
105 def feed(self, char):
106 if char is EOF:
107 return EOF, EOF
108 self.set_consumer(IgnoreCCode())
109
110 @staticmethod
111 def ignore(char):
112 return char in [' ', '\t', '\n']
113
114 @token(r'%(left|right|nonassoc)')
115 def assoc_decl(self, tok):
116 pass
117
118 @token(r'[a-zA-Z_][a-zA-Z0-9_]*')
119 def identifier(self, tok):
120 pass
121
122 @token('[1-9][0-9]*')
123 def number(self, tok):
124 tok.value = int(tok.value)
125
126 @token('"')
127 def string(self, tok):
128 class StringParser(object):
129 def __init__(self):
130 self.state = 0
131 self.value = io.StringIO()
132 def feed(self, char):
133 if self.state == 0:
134 if char == '"':
135 return 'string', self.value.getvalue()
136 if char == '\\':
137 self.state = 1
138 else:
139 self.value.write(char)
140 elif self.state == 1:
141 self.value.write(char)
142 self.state = 0
143 self.set_consumer(StringParser())
144
145 @token(r'\{')
146 def semantic_action(self, tok):
147 # Don't try to be too smart; just balance {} that are not in string litterals
148 class CSemanticAction(object):
149 def __init__(self):
150 self.state = 0
151 self.count = 1
152 self.value = io.StringIO()
153 self.value.write('{')
154
155 def feed(self, char):
156 self.value.write(char)
157 if self.state == 0: # Nothing special
158 if char == '}':
159 self.count -= 1
160 if self.count == 0:
161 return 'semantic_action', self.value.getvalue()
162 elif char == '{':
163 self.count += 1
164 elif char == '\\':
165 self.state = 1
166 elif char == '\'':
167 self.state = 2
168 elif char == '"':
169 self.state = 4
170 elif self.state == 1: # Escaping single char
171 self.state = 0
172 elif self.state == 2: # Character litteral. Not that this accepts several characters
173 if char == '\\':
174 self.state = 3
175 elif char == '\'':
176 self.state = 0
177 elif self.state == 3: # Escaping in character litteral
178 self.state = 2
179 elif self.state == 4: # In string litteral
180 if char == '\\':
181 self.state = 5
182 elif char == '"':
183 self.state = 0
184 elif self.state == 5: # Escaping in string litteral
185 self.state = 4
186 self.set_consumer(CSemanticAction())
187
188 @token(r'\'.\'')
189 def litteral_token(self, tok):
190 tok.value = tok.value[1]
191
192 # Parser
193
194 @production('YACC_FILE -> META_DECLARATION* part_sep PRODUCTION_DECL*')
195 def yacc_file(self):
196 pass
197
198 # Tokens, start symbol, etc
199
200 @production('META_DECLARATION -> "%token" identifier+<tokens>')
201 def token_declaration(self, tokens):
202 self.allTokens.extend(tokens)
203
204 @production('META_DECLARATION -> assoc_decl<assoc> identifier+<tokens>')
205 def assoc_declaration(self, assoc, tokens):
206 self.precedences.append((assoc, tokens))
207
208 @production('META_DECLARATION -> "%start" identifier<name>')
209 def start_declaration(self, name):
210 self.yaccStartSymbol = name
211
212 @production('META_DECLARATION -> "%type" identifier identifier+')
213 @production('META_DECLARATION -> "%expect" number')
214 @production('META_DECLARATION -> "%debug"')
215 @production('META_DECLARATION -> "%defines"')
216 @production('META_DECLARATION -> "%destructor" semantic_action identifier+')
217 @production('META_DECLARATION -> "%file-prefix" "=" string')
218 @production('META_DECLARATION -> "%locations"')
219 @production('META_DECLARATION -> "%name-prefix" "=" string')
220 @production('META_DECLARATION -> "%no-parser')
221 @production('META_DECLARATION -> "%no-lines')
222 @production('META_DECLARATION -> "%output" "=" string')
223 @production('META_DECLARATION -> "%pure-parser"')
224 @production('META_DECLARATION -> "%token-table"')
225 @production('META_DECLARATION -> "%verbose"')
226 @production('META_DECLARATION -> "%yacc"')
227 def ignored_declaration(self):
228 pass
229
230 # Productions
231
232 @production('PRODUCTION_DECL -> identifier<left> ":" PRODUCTION_RIGHT+("|")<right> ";"')
233 def production_decl(self, left, right):
234 self.allProductions.append((left, right))
235
236 @production('PRODUCTION_RIGHT -> SYMBOL*<symbols>')
237 def production_right(self, symbols):
238 names = list()
239 indexes = dict()
240 for symbol in symbols:
241 if symbol.argname is None:
242 names.append((symbol.name, None))
243 else:
244 index = indexes.get(symbol.argname, 0)
245 argname = symbol.argname if index == 0 else '%s_%d' % (symbol.argname, index + 1)
246 indexes[symbol.argname] = index + 1
247 names.append((symbol.name, argname))
248
249 return dict(names=names, action=None, precedence=None)
250
251 @production('PRODUCTION_RIGHT -> PRODUCTION_RIGHT<prod> semantic_action<action>')
252 def production_right_action(self, prod, action):
253 if prod['action'] is not None:
254 raise RuntimeError('Duplicate semantic action "%s"' % action)
255 prod['action'] = action
256 return prod
257
258 @production('PRODUCTION_RIGHT -> PRODUCTION_RIGHT<prod> "%prec" identifier<prec>')
259 def production_right_prec(self, prod, prec):
260 if prod['precedence'] is not None:
261 raise RuntimeError('Duplicate precedence declaration "%s"' % prec)
262 prod['precedence'] = prec
263 return prod
264
265 @production('SYMBOL -> identifier<tok>')
266 def symbol_from_identifier(self, tok):
267 return Symbol(tok, None if tok in self.allTokens else tok)
268
269 @production('SYMBOL -> litteral_token<tok>')
270 def symbol_from_litteral(self, tok):
271 return Symbol('"%s"' % tok, None)
272
273 def on_start_symbol(self, result):
274 self.stream.write('from ptk.lexer import ReLexer, token\n')
275 self.stream.write('from ptk.parser import LRParser, production, left_assoc, right_assoc, non_assoc\n')
276 self.stream.write('\n')
277
278 for assocType, tokens in self.precedences:
279 self.stream.write('@%s(%s)\n' % ({'%left': 'left_assoc', '%right': 'right_assoc', '%nonassoc': 'non_assoc'}[assocType],
280 ', '.join([repr(tok) for tok in tokens])))
281 self.stream.write('class Parser(LRParser, ReLexer):\n')
282 if self.yaccStartSymbol is not None:
283 self.stream.write(' startSymbol = %s\n' % repr(self.yaccStartSymbol))
284 self.stream.write('\n')
285
286 self.stream.write(' # Lexer\n')
287 for name in self.allTokens:
288 self.stream.write('\n')
289 self.stream.write(' @token(r\'\')\n')
290 self.stream.write(' def %s(self, tok):\n' % name)
291 self.stream.write(' pass\n')
292
293 methodIndexes = dict()
294 def methodName(name):
295 index = methodIndexes.get(name, 0)
296 methodIndexes[name] = index + 1
297 return name if index == 0 else '%s_%d' % (name, index + 1)
298
299 for name, prods in self.allProductions:
300 for prod in prods:
301 if not self.options.compact:
302 self.stream.write('\n')
303 if prod['action'] is not None:
304 for line in prod['action'].split('\n'):
305 self.stream.write(' # %s\n' % line)
306 symnames = []
307 for aname, argname in prod['names']:
308 symnames.append(aname if argname is None or not self.options.arguments else '%s<%s>' % (aname, argname))
309 self.stream.write(' @production(\'%s -> %s\'' % (name, ' '.join(symnames)))
310 if prod['precedence'] is not None:
311 self.stream.write(', priority=%s' % repr(prod['precedence']))
312 self.stream.write(')\n')
313 if not self.options.compact:
314 self.stream.write(' def %s(self' % methodName(name))
315 if self.options.arguments:
316 for aname, argname in prod['names']:
317 if argname is not None:
318 self.stream.write(', %s' % argname)
319 self.stream.write('):\n')
320 self.stream.write(' pass\n')
321 if self.options.compact:
322 self.stream.write(' def %s(self):\n' % methodName(name))
323 self.stream.write(' pass\n')
324 self.stream.write('\n')
325
326
327 def main(filename):
328 import time
329 options, filenames = Options.create()
330 for filename in filenames:
331 with codecs.getreader('utf_8')(open(filename, 'rb')) as fileobj:
332 output = sys.stdout if options.filename == '-' else codecs.getwriter('utf_8')(open(options.filename, 'wb'))
333 parser = YaccParser(options, output)
334 t0 = time.time()
335 try:
336 parser.parse(fileobj.read())
337 except ParseError as exc:
338 print('Parse error: %s' % exc)
339 tokens = exc.expecting()
340 if tokens:
341 print('Was expecting %s' % ', '.join(map(repr, sorted(tokens))))
342 sys.exit(1)
343 finally:
344 print('== Parsed file in %d ms.' % int(1000 * (time.time() - t0)))
345
346
347 if __name__ == '__main__':
348 import logging
349 logging.basicConfig(level=logging.WARNING, format='%(asctime)-15s %(levelname)-8s %(name)-15s %(message)s')
350
351 import sys
352 main(sys.argv[1])