Sync with master

This commit is contained in:
rocky
2017-10-10 22:50:40 -04:00
parent c796d6a799
commit 9e0c65881d
30 changed files with 215 additions and 197 deletions

View File

@@ -89,17 +89,14 @@ class PythonParser(GenericASTBuilder):
for i in dir(self):
setattr(self, i, None)
def debug_reduce(self, rule, tokens, parent, i):
def debug_reduce(self, rule, tokens, parent, last_token_pos):
"""Customized format and print for our kind of tokens
which gets called in debugging grammar reduce rules
"""
def fix(c):
s = str(c)
i = s.find('_')
if i == -1:
return s
else:
return s[:i]
last_token_pos = s.find('_')
return s if last_token_pos == -1 else s[:last_token_pos]
prefix = ''
if parent and tokens:
@@ -111,13 +108,13 @@ class PythonParser(GenericASTBuilder):
if hasattr(p_token, 'offset'):
prefix += "%3s" % fix(p_token.offset)
if len(rule[1]) > 1:
prefix += '-%-3s ' % fix(tokens[i-1].offset)
prefix += '-%-3s ' % fix(tokens[last_token_pos-1].offset)
else:
prefix += ' '
else:
prefix = ' '
print("%s%s ::= %s" % (prefix, rule[0], ' '.join(rule[1])))
print("%s%s ::= %s (%d)" % (prefix, rule[0], ' '.join(rule[1]), last_token_pos))
def error(self, instructions, index):
# Find the last line boundary
@@ -138,7 +135,7 @@ class PythonParser(GenericASTBuilder):
raise ParserError(err_token, err_token.offset)
def typestring(self, token):
return token.type
return token.kind
def nonterminal(self, nt, args):
if nt in self.collect and len(args) > 1:
@@ -737,7 +734,7 @@ def get_python_parser(
else:
p = parse3.Python3ParserSingle(debug_parser)
p.version = version
# p.dumpGrammar() # debug
# p.dump_grammar() # debug
return p
class PythonParserSingle(PythonParser):