| 1 |
#!/usr/bin/env python3 |
| 2 |
# -*- coding: utf-8 -*- |
| 3 |
|
| 4 |
""" |
| 5 |
Converts a Yacc/Bison grammar definition into a Python skeleton that uses ptk. |
| 6 |
""" |
| 7 |
|
| 8 |
import io |
| 9 |
import getopt |
| 10 |
import sys |
| 11 |
import collections |
| 12 |
import codecs |
| 13 |
import re |
| 14 |
|
| 15 |
from ptk.parser import production, LRParser, ParseError |
| 16 |
from ptk.lexer import token, ReLexer, EOF |
| 17 |
from ptk.regex import buildRegex, DeadState |
| 18 |
|
| 19 |
|
| 20 |
Symbol = collections.namedtuple('Symbol', ('name', 'argname')) |
| 21 |
|
| 22 |
|
| 23 |
class Options(object): |
| 24 |
def __init__(self, opts): |
| 25 |
self.compact = False |
| 26 |
self.arguments = False |
| 27 |
self.filename = None |
| 28 |
for opt, val in opts: |
| 29 |
if opt in ('-c', '--compact'): |
| 30 |
self.compact = True |
| 31 |
if opt in ('-a', '--arguments'): |
| 32 |
self.arguments = True |
| 33 |
if opt in ('-o', '--output'): |
| 34 |
self.filename = val |
| 35 |
if opt in ('-h', '--help'): |
| 36 |
self.usage() |
| 37 |
|
| 38 |
if self.compact and self.arguments: |
| 39 |
print('--compact and --arguments are not compatible') |
| 40 |
self.usage(1) |
| 41 |
|
| 42 |
if self.filename is None: |
| 43 |
print('Output file not specified') |
| 44 |
self.usage(1) |
| 45 |
|
| 46 |
def usage(self, exitCode=0): |
| 47 |
print('Usage: %s [options] filename' % sys.argv[0]) |
| 48 |
print('Options:') |
| 49 |
print(' -h, --help Print this') |
| 50 |
print(' -c, --compact Create one method for all alternatives of a production') |
| 51 |
print(' -o, --output <filename> Output to file (mandatory)') |
| 52 |
print(' -a, --arguments Generate argument names for items in productions (incompatible with --compact)') |
| 53 |
sys.exit(exitCode) |
| 54 |
|
| 55 |
@staticmethod |
| 56 |
def create(): |
| 57 |
opts, args = getopt.getopt(sys.argv[1:], 'caho:', ['compact', 'arguments', 'help', 'output=']) |
| 58 |
return Options(opts), args |
| 59 |
|
| 60 |
|
| 61 |
class NullToken(object): |
| 62 |
def __init__(self, endMarker): |
| 63 |
self.__rx = buildRegex('(.|\n)*%s' % re.escape(endMarker)).start() |
| 64 |
|
| 65 |
def feed(self, char): |
| 66 |
try: |
| 67 |
if self.__rx.feed(char): |
| 68 |
return None, None |
| 69 |
except DeadState: |
| 70 |
return None, None |
| 71 |
|
| 72 |
|
| 73 |
class YaccParser(LRParser, ReLexer): |
| 74 |
def __init__(self, options, stream): |
| 75 |
self.stream = stream |
| 76 |
self.options = options |
| 77 |
super().__init__() |
| 78 |
|
| 79 |
self.state = 0 |
| 80 |
self.yaccStartSymbol = None |
| 81 |
self.allTokens = list() |
| 82 |
self.allProductions = list() |
| 83 |
self.precedences = list() |
| 84 |
|
| 85 |
# Lexer |
| 86 |
|
| 87 |
@token(r'%\{', types=[]) |
| 88 |
def c_decl(self, tok): |
| 89 |
self.set_consumer(NullToken('%}')) |
| 90 |
|
| 91 |
@token(r'/\*', types=[]) |
| 92 |
def comment(self, tok): |
| 93 |
self.set_consumer(NullToken('*/')) |
| 94 |
|
| 95 |
@token(r'%union\s*{', types=[]) # Hum, no LF possible before { |
| 96 |
def union(self, tok): |
| 97 |
self.set_consumer(NullToken('}')) |
| 98 |
|
| 99 |
@token(r'%%') |
| 100 |
def part_sep(self, tok): |
| 101 |
self.state += 1 |
| 102 |
if self.state == 2: |
| 103 |
# Ignore C code after last %% |
| 104 |
class IgnoreCCode(object): |
| 105 |
def feed(self, char): |
| 106 |
if char is EOF: |
| 107 |
return EOF, EOF |
| 108 |
self.set_consumer(IgnoreCCode()) |
| 109 |
|
| 110 |
@staticmethod |
| 111 |
def ignore(char): |
| 112 |
return char in [' ', '\t', '\n'] |
| 113 |
|
| 114 |
@token(r'%(left|right|nonassoc)') |
| 115 |
def assoc_decl(self, tok): |
| 116 |
pass |
| 117 |
|
| 118 |
@token(r'[a-zA-Z_][a-zA-Z0-9_]*') |
| 119 |
def identifier(self, tok): |
| 120 |
pass |
| 121 |
|
| 122 |
@token('[1-9][0-9]*') |
| 123 |
def number(self, tok): |
| 124 |
tok.value = int(tok.value) |
| 125 |
|
| 126 |
@token('"') |
| 127 |
def string(self, tok): |
| 128 |
class StringParser(object): |
| 129 |
def __init__(self): |
| 130 |
self.state = 0 |
| 131 |
self.value = io.StringIO() |
| 132 |
def feed(self, char): |
| 133 |
if self.state == 0: |
| 134 |
if char == '"': |
| 135 |
return 'string', self.value.getvalue() |
| 136 |
if char == '\\': |
| 137 |
self.state = 1 |
| 138 |
else: |
| 139 |
self.value.write(char) |
| 140 |
elif self.state == 1: |
| 141 |
self.value.write(char) |
| 142 |
self.state = 0 |
| 143 |
self.set_consumer(StringParser()) |
| 144 |
|
| 145 |
@token(r'\{') |
| 146 |
def semantic_action(self, tok): |
| 147 |
# Don't try to be too smart; just balance {} that are not in string litterals |
| 148 |
class CSemanticAction(object): |
| 149 |
def __init__(self): |
| 150 |
self.state = 0 |
| 151 |
self.count = 1 |
| 152 |
self.value = io.StringIO() |
| 153 |
self.value.write('{') |
| 154 |
|
| 155 |
def feed(self, char): |
| 156 |
self.value.write(char) |
| 157 |
if self.state == 0: # Nothing special |
| 158 |
if char == '}': |
| 159 |
self.count -= 1 |
| 160 |
if self.count == 0: |
| 161 |
return 'semantic_action', self.value.getvalue() |
| 162 |
elif char == '{': |
| 163 |
self.count += 1 |
| 164 |
elif char == '\\': |
| 165 |
self.state = 1 |
| 166 |
elif char == '\'': |
| 167 |
self.state = 2 |
| 168 |
elif char == '"': |
| 169 |
self.state = 4 |
| 170 |
elif self.state == 1: # Escaping single char |
| 171 |
self.state = 0 |
| 172 |
elif self.state == 2: # Character litteral. Not that this accepts several characters |
| 173 |
if char == '\\': |
| 174 |
self.state = 3 |
| 175 |
elif char == '\'': |
| 176 |
self.state = 0 |
| 177 |
elif self.state == 3: # Escaping in character litteral |
| 178 |
self.state = 2 |
| 179 |
elif self.state == 4: # In string litteral |
| 180 |
if char == '\\': |
| 181 |
self.state = 5 |
| 182 |
elif char == '"': |
| 183 |
self.state = 0 |
| 184 |
elif self.state == 5: # Escaping in string litteral |
| 185 |
self.state = 4 |
| 186 |
self.set_consumer(CSemanticAction()) |
| 187 |
|
| 188 |
@token(r'\'.\'') |
| 189 |
def litteral_token(self, tok): |
| 190 |
tok.value = tok.value[1] |
| 191 |
|
| 192 |
# Parser |
| 193 |
|
| 194 |
@production('YACC_FILE -> META_DECLARATION* part_sep PRODUCTION_DECL*') |
| 195 |
def yacc_file(self): |
| 196 |
pass |
| 197 |
|
| 198 |
# Tokens, start symbol, etc |
| 199 |
|
| 200 |
@production('META_DECLARATION -> "%token" identifier+<tokens>') |
| 201 |
def token_declaration(self, tokens): |
| 202 |
self.allTokens.extend(tokens) |
| 203 |
|
| 204 |
@production('META_DECLARATION -> assoc_decl<assoc> identifier+<tokens>') |
| 205 |
def assoc_declaration(self, assoc, tokens): |
| 206 |
self.precedences.append((assoc, tokens)) |
| 207 |
|
| 208 |
@production('META_DECLARATION -> "%start" identifier<name>') |
| 209 |
def start_declaration(self, name): |
| 210 |
self.yaccStartSymbol = name |
| 211 |
|
| 212 |
@production('META_DECLARATION -> "%type" identifier identifier+') |
| 213 |
@production('META_DECLARATION -> "%expect" number') |
| 214 |
@production('META_DECLARATION -> "%debug"') |
| 215 |
@production('META_DECLARATION -> "%defines"') |
| 216 |
@production('META_DECLARATION -> "%destructor" semantic_action identifier+') |
| 217 |
@production('META_DECLARATION -> "%file-prefix" "=" string') |
| 218 |
@production('META_DECLARATION -> "%locations"') |
| 219 |
@production('META_DECLARATION -> "%name-prefix" "=" string') |
| 220 |
@production('META_DECLARATION -> "%no-parser') |
| 221 |
@production('META_DECLARATION -> "%no-lines') |
| 222 |
@production('META_DECLARATION -> "%output" "=" string') |
| 223 |
@production('META_DECLARATION -> "%pure-parser"') |
| 224 |
@production('META_DECLARATION -> "%token-table"') |
| 225 |
@production('META_DECLARATION -> "%verbose"') |
| 226 |
@production('META_DECLARATION -> "%yacc"') |
| 227 |
def ignored_declaration(self): |
| 228 |
pass |
| 229 |
|
| 230 |
# Productions |
| 231 |
|
| 232 |
@production('PRODUCTION_DECL -> identifier<left> ":" PRODUCTION_RIGHT+("|")<right> ";"') |
| 233 |
def production_decl(self, left, right): |
| 234 |
self.allProductions.append((left, right)) |
| 235 |
|
| 236 |
@production('PRODUCTION_RIGHT -> SYMBOL*<symbols>') |
| 237 |
def production_right(self, symbols): |
| 238 |
names = list() |
| 239 |
indexes = dict() |
| 240 |
for symbol in symbols: |
| 241 |
if symbol.argname is None: |
| 242 |
names.append((symbol.name, None)) |
| 243 |
else: |
| 244 |
index = indexes.get(symbol.argname, 0) |
| 245 |
argname = symbol.argname if index == 0 else '%s_%d' % (symbol.argname, index + 1) |
| 246 |
indexes[symbol.argname] = index + 1 |
| 247 |
names.append((symbol.name, argname)) |
| 248 |
|
| 249 |
return dict(names=names, action=None, precedence=None) |
| 250 |
|
| 251 |
@production('PRODUCTION_RIGHT -> PRODUCTION_RIGHT<prod> semantic_action<action>') |
| 252 |
def production_right_action(self, prod, action): |
| 253 |
if prod['action'] is not None: |
| 254 |
raise RuntimeError('Duplicate semantic action "%s"' % action) |
| 255 |
prod['action'] = action |
| 256 |
return prod |
| 257 |
|
| 258 |
@production('PRODUCTION_RIGHT -> PRODUCTION_RIGHT<prod> "%prec" identifier<prec>') |
| 259 |
def production_right_prec(self, prod, prec): |
| 260 |
if prod['precedence'] is not None: |
| 261 |
raise RuntimeError('Duplicate precedence declaration "%s"' % prec) |
| 262 |
prod['precedence'] = prec |
| 263 |
return prod |
| 264 |
|
| 265 |
@production('SYMBOL -> identifier<tok>') |
| 266 |
def symbol_from_identifier(self, tok): |
| 267 |
return Symbol(tok, None if tok in self.allTokens else tok) |
| 268 |
|
| 269 |
@production('SYMBOL -> litteral_token<tok>') |
| 270 |
def symbol_from_litteral(self, tok): |
| 271 |
return Symbol('"%s"' % tok, None) |
| 272 |
|
| 273 |
def on_start_symbol(self, result): |
| 274 |
self.stream.write('from ptk.lexer import ReLexer, token\n') |
| 275 |
self.stream.write('from ptk.parser import LRParser, production, left_assoc, right_assoc, non_assoc\n') |
| 276 |
self.stream.write('\n') |
| 277 |
|
| 278 |
for assocType, tokens in self.precedences: |
| 279 |
self.stream.write('@%s(%s)\n' % ({'%left': 'left_assoc', '%right': 'right_assoc', '%nonassoc': 'non_assoc'}[assocType], |
| 280 |
', '.join([repr(tok) for tok in tokens]))) |
| 281 |
self.stream.write('class Parser(LRParser, ReLexer):\n') |
| 282 |
if self.yaccStartSymbol is not None: |
| 283 |
self.stream.write(' startSymbol = %s\n' % repr(self.yaccStartSymbol)) |
| 284 |
self.stream.write('\n') |
| 285 |
|
| 286 |
self.stream.write(' # Lexer\n') |
| 287 |
for name in self.allTokens: |
| 288 |
self.stream.write('\n') |
| 289 |
self.stream.write(' @token(r\'\')\n') |
| 290 |
self.stream.write(' def %s(self, tok):\n' % name) |
| 291 |
self.stream.write(' pass\n') |
| 292 |
|
| 293 |
methodIndexes = dict() |
| 294 |
def methodName(name): |
| 295 |
index = methodIndexes.get(name, 0) |
| 296 |
methodIndexes[name] = index + 1 |
| 297 |
return name if index == 0 else '%s_%d' % (name, index + 1) |
| 298 |
|
| 299 |
for name, prods in self.allProductions: |
| 300 |
for prod in prods: |
| 301 |
if not self.options.compact: |
| 302 |
self.stream.write('\n') |
| 303 |
if prod['action'] is not None: |
| 304 |
for line in prod['action'].split('\n'): |
| 305 |
self.stream.write(' # %s\n' % line) |
| 306 |
symnames = [] |
| 307 |
for aname, argname in prod['names']: |
| 308 |
symnames.append(aname if argname is None or not self.options.arguments else '%s<%s>' % (aname, argname)) |
| 309 |
self.stream.write(' @production(\'%s -> %s\'' % (name, ' '.join(symnames))) |
| 310 |
if prod['precedence'] is not None: |
| 311 |
self.stream.write(', priority=%s' % repr(prod['precedence'])) |
| 312 |
self.stream.write(')\n') |
| 313 |
if not self.options.compact: |
| 314 |
self.stream.write(' def %s(self' % methodName(name)) |
| 315 |
if self.options.arguments: |
| 316 |
for aname, argname in prod['names']: |
| 317 |
if argname is not None: |
| 318 |
self.stream.write(', %s' % argname) |
| 319 |
self.stream.write('):\n') |
| 320 |
self.stream.write(' pass\n') |
| 321 |
if self.options.compact: |
| 322 |
self.stream.write(' def %s(self):\n' % methodName(name)) |
| 323 |
self.stream.write(' pass\n') |
| 324 |
self.stream.write('\n') |
| 325 |
|
| 326 |
|
| 327 |
def main(filename): |
| 328 |
import time |
| 329 |
options, filenames = Options.create() |
| 330 |
for filename in filenames: |
| 331 |
with codecs.getreader('utf_8')(open(filename, 'rb')) as fileobj: |
| 332 |
output = sys.stdout if options.filename == '-' else codecs.getwriter('utf_8')(open(options.filename, 'wb')) |
| 333 |
parser = YaccParser(options, output) |
| 334 |
t0 = time.time() |
| 335 |
try: |
| 336 |
parser.parse(fileobj.read()) |
| 337 |
except ParseError as exc: |
| 338 |
print('Parse error: %s' % exc) |
| 339 |
tokens = exc.expecting() |
| 340 |
if tokens: |
| 341 |
print('Was expecting %s' % ', '.join(map(repr, sorted(tokens)))) |
| 342 |
sys.exit(1) |
| 343 |
finally: |
| 344 |
print('== Parsed file in %d ms.' % int(1000 * (time.time() - t0))) |
| 345 |
|
| 346 |
|
| 347 |
if __name__ == '__main__': |
| 348 |
import logging |
| 349 |
logging.basicConfig(level=logging.WARNING, format='%(asctime)-15s %(levelname)-8s %(name)-15s %(message)s') |
| 350 |
|
| 351 |
import sys |
| 352 |
main(sys.argv[1]) |