Start grammar reduction checks

__pkginfo__.py

@@ -37,7 +37,7 @@ entry_points={
         'pydisassemble=uncompyle6.bin.pydisassemble:main',
     ]}
 ftp_url            = None
-install_requires   = ['spark-parser >= 1.4.3, < 1.5.0',
+install_requires   = ['spark-parser >= 1.5.0, < 1.6.0',
                       'xdis >= 3.2.3, < 3.3.0']
 license            = 'MIT'
 mailing_list       = 'python-debugger@googlegroups.com'

test/Makefile

@@ -104,7 +104,7 @@ check-bytecode-2.6:
 
 #: Check deparsing Python 2.7
 check-bytecode-2.7:
-	$(PYTHON) test_pythonlib.py --bytecode-2.7
+	$(PYTHON) test_pythonlib.py --bytecode-2.7 --verify
 
 #: Check deparsing Python 3.0
 check-bytecode-3.0:

uncompyle6/parser.py

@@ -69,6 +69,25 @@ class PythonParser(GenericASTBuilder):
         for i in dir(self):
             setattr(self, i, None)
 
+    def debug_reduce(self, rule, tokens, parent, i):
+        """Customized format and print for our kind of tokens
+        which gets called in debugging grammar reduce rules
+        """
+        prefix = ''
+        if parent and tokens:
+            p_token = tokens[parent]
+            if hasattr(p_token, 'linestart') and p_token.linestart:
+                prefix = 'L.%3d: ' % p_token.linestart
+            else:
+                prefix = '       '
+            if hasattr(p_token, 'offset'):
+                prefix += "%3d " % p_token.offset
+                prefix += "    "
+        else:
+            prefix = '               '
+
+        print("%s%s ::= %s" % (prefix, rule[0], ' '.join(rule[1])))
+
     def error(self, instructions, index):
         # Find the last line boundary
         for start in range(index, -1, -1):

uncompyle6/parsers/parse2.py

@@ -241,7 +241,7 @@ class Python2Parser(PythonParser):
         """
 
     def add_custom_rules(self, tokens, customize):
-        '''
+        """
         Special handling for opcodes such as those that take a variable number
         of arguments -- we add a new rule for each:
 
@@ -260,7 +260,7 @@ class Python2Parser(PythonParser):
             expr ::= expr {expr}^n CALL_FUNCTION_KW_n POP_TOP
 
         PyPy adds custom rules here as well
-        '''
+        """
         for opname, v in list(customize.items()):
             opname_base = opname[:opname.rfind('_')]
             if opname == 'PyPy':
@@ -389,6 +389,17 @@ class Python2Parser(PythonParser):
             else:
                 raise Exception('unknown customize token %s' % opname)
             self.add_unique_rule(rule, opname_base, v, customize)
+            pass
+        self.check_reduce['augassign1'] = 'AST'
+        self.check_reduce['augassign2'] = 'AST'
+        return
+
+    def reduce_is_invalid(self, rule, ast, tokens, first, last):
+        lhs = rule[0]
+        if lhs in ('augassign1', 'augassign2') and ast[0][0] == 'and':
+            return True
+        # Add more stuff, like COME_FROM checking
+        return False
 
 class Python2ParserSingle(Python2Parser, PythonParserSingle):
     pass

uncompyle6/parsers/parse3.py

@@ -146,8 +146,6 @@ class Python3Parser(PythonParser):
         ifelsestmtr ::= testexpr return_if_stmts return_stmts
 
         ifelsestmtl ::= testexpr c_stmts_opt JUMP_BACK else_suitel
-        ifelsestmtl ::= testexpr c_stmts_opt JUMP_BACK else_suitel JUMP_BACK COME_FROM_LOOP
-        ifelsestmtl ::= testexpr c_stmts_opt JUMP_BACK else_suitel COME_FROM_LOOP
 
 
         # FIXME: this feels like a hack. Is it just 1 or two
@@ -335,9 +333,6 @@ class Python3Parser(PythonParser):
         whilestmt         ::= SETUP_LOOP testexpr return_stmts          POP_BLOCK
                               COME_FROM_LOOP
 
-        whileelsestmt     ::= SETUP_LOOP testexpr l_stmts_opt JUMP_BACK POP_BLOCK
-                              else_suite COME_FROM_LOOP
-
         while1elsestmt    ::= SETUP_LOOP          l_stmts     JUMP_BACK
                               else_suite
 
@@ -348,6 +343,7 @@ class Python3Parser(PythonParser):
 
         # FIXME: Python 3.? starts adding branch optimization? Put this starting there.
         while1stmt        ::= SETUP_LOOP l_stmts
+        while1stmt        ::= SETUP_LOOP l_stmts COME_FROM_LOOP
 
         # FIXME: investigate - can code really produce a NOP?
         whileTruestmt     ::= SETUP_LOOP l_stmts_opt JUMP_BACK NOP
@@ -680,8 +676,31 @@ class Python3Parser(PythonParser):
                 rule = ('mkfunc ::= %sload_closure LOAD_CONST %s'
                         % ('expr ' * args_pos, opname))
                 self.add_unique_rule(rule, opname, token.attr, customize)
+                pass
+        self.check_reduce['augassign1'] = 'AST'
+        self.check_reduce['augassign2'] = 'AST'
+        self.check_reduce['while1stmt'] = 'noAST'
         return
 
+    def reduce_is_invalid(self, rule, ast, tokens, first, last):
+        lhs = rule[0]
+        if lhs in ('augassign1', 'augassign2') and ast[0][0] == 'and':
+            return True
+        elif lhs == 'while1stmt':
+            # Skip COME_FROM tokens
+            skip = 0
+            if tokens[last] != 'COME_FROM_LOOP':
+                skip = 1
+            while last+skip < len(tokens) and isinstance(tokens[last+skip].offset, str):
+                last += 1
+            if last + skip < len(tokens):
+                offset = tokens[last+skip].offset
+                assert tokens[first] == 'SETUP_LOOP'
+                if offset != tokens[first].attr:
+                    return True
+            return False
+        return False
+
 class Python30Parser(Python3Parser):
 
     def p_30(self, args):