From 7ab4e1fbdb420df79ce4feeeaa39b945d7dca325 Mon Sep 17 00:00:00 2001 From: rocky Date: Fri, 25 Nov 2016 12:30:42 -0500 Subject: [PATCH] Start grammar reduction checks --- __pkginfo__.py | 2 +- test/Makefile | 2 +- ...le1.pyc => 02_while1_if_while1.pyc-notyet} | Bin ...while1.pyc => 04_while1_while1.pyc-notyet} | Bin uncompyle6/parser.py | 19 ++++++++++++ uncompyle6/parsers/parse2.py | 15 +++++++-- uncompyle6/parsers/parse3.py | 29 +++++++++++++++--- 7 files changed, 58 insertions(+), 9 deletions(-) rename test/bytecode_3.0/{02_while1_if_while1.pyc => 02_while1_if_while1.pyc-notyet} (100%) rename test/bytecode_3.4/{04_while1_while1.pyc => 04_while1_while1.pyc-notyet} (100%) diff --git a/__pkginfo__.py b/__pkginfo__.py index 7ee929f1..076581bc 100644 --- a/__pkginfo__.py +++ b/__pkginfo__.py @@ -37,7 +37,7 @@ entry_points={ 'pydisassemble=uncompyle6.bin.pydisassemble:main', ]} ftp_url = None -install_requires = ['spark-parser >= 1.4.3, < 1.5.0', +install_requires = ['spark-parser >= 1.5.0, < 1.6.0', 'xdis >= 3.2.3, < 3.3.0'] license = 'MIT' mailing_list = 'python-debugger@googlegroups.com' diff --git a/test/Makefile b/test/Makefile index 2f8e56ca..343d4c19 100644 --- a/test/Makefile +++ b/test/Makefile @@ -104,7 +104,7 @@ check-bytecode-2.6: #: Check deparsing Python 2.7 check-bytecode-2.7: - $(PYTHON) test_pythonlib.py --bytecode-2.7 + $(PYTHON) test_pythonlib.py --bytecode-2.7 --verify #: Check deparsing Python 3.0 check-bytecode-3.0: diff --git a/test/bytecode_3.0/02_while1_if_while1.pyc b/test/bytecode_3.0/02_while1_if_while1.pyc-notyet similarity index 100% rename from test/bytecode_3.0/02_while1_if_while1.pyc rename to test/bytecode_3.0/02_while1_if_while1.pyc-notyet diff --git a/test/bytecode_3.4/04_while1_while1.pyc b/test/bytecode_3.4/04_while1_while1.pyc-notyet similarity index 100% rename from test/bytecode_3.4/04_while1_while1.pyc rename to test/bytecode_3.4/04_while1_while1.pyc-notyet diff --git a/uncompyle6/parser.py b/uncompyle6/parser.py index ea730dfc..ee9abcb1 100644 --- a/uncompyle6/parser.py +++ b/uncompyle6/parser.py @@ -69,6 +69,25 @@ class PythonParser(GenericASTBuilder): for i in dir(self): setattr(self, i, None) + def debug_reduce(self, rule, tokens, parent, i): + """Customized format and print for our kind of tokens + which gets called in debugging grammar reduce rules + """ + prefix = '' + if parent and tokens: + p_token = tokens[parent] + if hasattr(p_token, 'linestart') and p_token.linestart: + prefix = 'L.%3d: ' % p_token.linestart + else: + prefix = ' ' + if hasattr(p_token, 'offset'): + prefix += "%3d " % p_token.offset + prefix += " " + else: + prefix = ' ' + + print("%s%s ::= %s" % (prefix, rule[0], ' '.join(rule[1]))) + def error(self, instructions, index): # Find the last line boundary for start in range(index, -1, -1): diff --git a/uncompyle6/parsers/parse2.py b/uncompyle6/parsers/parse2.py index 530c3585..6778f014 100644 --- a/uncompyle6/parsers/parse2.py +++ b/uncompyle6/parsers/parse2.py @@ -241,7 +241,7 @@ class Python2Parser(PythonParser): """ def add_custom_rules(self, tokens, customize): - ''' + """ Special handling for opcodes such as those that take a variable number of arguments -- we add a new rule for each: @@ -260,7 +260,7 @@ class Python2Parser(PythonParser): expr ::= expr {expr}^n CALL_FUNCTION_KW_n POP_TOP PyPy adds custom rules here as well - ''' + """ for opname, v in list(customize.items()): opname_base = opname[:opname.rfind('_')] if opname == 'PyPy': @@ -389,6 +389,17 @@ class Python2Parser(PythonParser): else: raise Exception('unknown customize token %s' % opname) self.add_unique_rule(rule, opname_base, v, customize) + pass + self.check_reduce['augassign1'] = 'AST' + self.check_reduce['augassign2'] = 'AST' + return + + def reduce_is_invalid(self, rule, ast, tokens, first, last): + lhs = rule[0] + if lhs in ('augassign1', 'augassign2') and ast[0][0] == 'and': + return True + # Add more stuff, like COME_FROM checking + return False class Python2ParserSingle(Python2Parser, PythonParserSingle): pass diff --git a/uncompyle6/parsers/parse3.py b/uncompyle6/parsers/parse3.py index adc52a86..0c1e7112 100644 --- a/uncompyle6/parsers/parse3.py +++ b/uncompyle6/parsers/parse3.py @@ -146,8 +146,6 @@ class Python3Parser(PythonParser): ifelsestmtr ::= testexpr return_if_stmts return_stmts ifelsestmtl ::= testexpr c_stmts_opt JUMP_BACK else_suitel - ifelsestmtl ::= testexpr c_stmts_opt JUMP_BACK else_suitel JUMP_BACK COME_FROM_LOOP - ifelsestmtl ::= testexpr c_stmts_opt JUMP_BACK else_suitel COME_FROM_LOOP # FIXME: this feels like a hack. Is it just 1 or two @@ -335,9 +333,6 @@ class Python3Parser(PythonParser): whilestmt ::= SETUP_LOOP testexpr return_stmts POP_BLOCK COME_FROM_LOOP - whileelsestmt ::= SETUP_LOOP testexpr l_stmts_opt JUMP_BACK POP_BLOCK - else_suite COME_FROM_LOOP - while1elsestmt ::= SETUP_LOOP l_stmts JUMP_BACK else_suite @@ -348,6 +343,7 @@ class Python3Parser(PythonParser): # FIXME: Python 3.? starts adding branch optimization? Put this starting there. while1stmt ::= SETUP_LOOP l_stmts + while1stmt ::= SETUP_LOOP l_stmts COME_FROM_LOOP # FIXME: investigate - can code really produce a NOP? whileTruestmt ::= SETUP_LOOP l_stmts_opt JUMP_BACK NOP @@ -680,8 +676,31 @@ class Python3Parser(PythonParser): rule = ('mkfunc ::= %sload_closure LOAD_CONST %s' % ('expr ' * args_pos, opname)) self.add_unique_rule(rule, opname, token.attr, customize) + pass + self.check_reduce['augassign1'] = 'AST' + self.check_reduce['augassign2'] = 'AST' + self.check_reduce['while1stmt'] = 'noAST' return + def reduce_is_invalid(self, rule, ast, tokens, first, last): + lhs = rule[0] + if lhs in ('augassign1', 'augassign2') and ast[0][0] == 'and': + return True + elif lhs == 'while1stmt': + # Skip COME_FROM tokens + skip = 0 + if tokens[last] != 'COME_FROM_LOOP': + skip = 1 + while last+skip < len(tokens) and isinstance(tokens[last+skip].offset, str): + last += 1 + if last + skip < len(tokens): + offset = tokens[last+skip].offset + assert tokens[first] == 'SETUP_LOOP' + if offset != tokens[first].attr: + return True + return False + return False + class Python30Parser(Python3Parser): def p_30(self, args):