Add python 3.4 grammar checking

DRY grammar testing

pytest/test_grammar.py

@@ -1,9 +1,18 @@
-import pytest
+import pytest, re
 from uncompyle6 import PYTHON_VERSION, PYTHON3, IS_PYPY # , PYTHON_VERSION
 from uncompyle6.parser import get_python_parser
 from uncompyle6.scanner import get_scanner
 
 def test_grammar():
+
+    def check_tokens(tokens, opcode_set):
+        remain_tokens = set(tokens) - opcode_set
+        remain_tokens = set([re.sub('_\d+$','', t) for t in remain_tokens])
+        remain_tokens = set([re.sub('_CONT$','', t) for t in remain_tokens])
+        remain_tokens = set(remain_tokens) - opcode_set
+        assert remain_tokens == set([]), \
+            "Remaining tokens %s\n====\n%s" % (remain_tokens, p.dumpGrammar())
+
     p = get_python_parser(PYTHON_VERSION, is_pypy=IS_PYPY)
     lhs, rhs, tokens, right_recursive = p.checkSets()
     expect_lhs = set(['expr1024', 'pos_arg'])
@@ -21,16 +30,15 @@ def test_grammar():
     assert unused_rhs == set(rhs)
     assert expect_right_recursive == right_recursive
     s = get_scanner(PYTHON_VERSION, IS_PYPY)
-    if PYTHON_VERSION == 2.7:
+    ignore_set = set(
-        opcode_set = set(s.opc.opname).union(set(
             """JUMP_BACK CONTINUE RETURN_END_IF COME_FROM
                LOAD_GENEXPR LOAD_ASSERT LOAD_SETCOMP LOAD_DICTCOMP
                LAMBDA_MARKER RETURN_LAST
-            """.split()))
+            """.split())
-        remain_tokens = set(tokens) - opcode_set
+    if 2.6 <= PYTHON_VERSION <= 2.7:
-        import re
+        opcode_set = set(s.opc.opname).union(ignore_set)
-        remain_tokens = set([re.sub('_\d+$','', t) for t in remain_tokens])
+        check_tokens(tokens, opcode_set)
-        remain_tokens = set([re.sub('_CONT$','', t) for t in remain_tokens])
+    elif PYTHON_VERSION == 3.4:
-        remain_tokens = set(remain_tokens) - opcode_set
+        ignore_set.add('LOAD_CLASSNAME')
-        assert remain_tokens == set([]), \
+        opcode_set = set(s.opc.opname).union(ignore_set)
-            "Remaining tokens %s\n====\n%s" % (remain_tokens, p.dumpGrammar())
+        check_tokens(tokens, opcode_set)

uncompyle6/parser.py

@@ -236,17 +236,12 @@ class PythonParser(GenericASTBuilder):
         stmt ::= augassign2
         augassign1 ::= expr expr inplace_op designator
         augassign1 ::= expr expr inplace_op ROT_THREE STORE_SUBSCR
-        augassign1 ::= expr expr inplace_op ROT_TWO   STORE_SLICE+0
-        augassign1 ::= expr expr inplace_op ROT_THREE STORE_SLICE+1
-        augassign1 ::= expr expr inplace_op ROT_THREE STORE_SLICE+2
-        augassign1 ::= expr expr inplace_op ROT_FOUR  STORE_SLICE+3
         augassign2 ::= expr DUP_TOP LOAD_ATTR expr
                 inplace_op ROT_TWO   STORE_ATTR
 
         inplace_op ::= INPLACE_ADD
         inplace_op ::= INPLACE_SUBTRACT
         inplace_op ::= INPLACE_MULTIPLY
-        inplace_op ::= INPLACE_DIVIDE
         inplace_op ::= INPLACE_TRUE_DIVIDE
         inplace_op ::= INPLACE_FLOOR_DIVIDE
         inplace_op ::= INPLACE_MODULO
@@ -405,15 +400,10 @@ class PythonParser(GenericASTBuilder):
         expr ::= unary_expr
         expr ::= call_function
         expr ::= unary_not
-        expr ::= unary_convert
         expr ::= binary_subscr
         expr ::= binary_subscr2
         expr ::= load_attr
         expr ::= get_iter
-        expr ::= slice0
-        expr ::= slice1
-        expr ::= slice2
-        expr ::= slice3
         expr ::= buildslice2
         expr ::= buildslice3
         expr ::= yield
@@ -428,7 +418,6 @@ class PythonParser(GenericASTBuilder):
         binary_op ::= BINARY_OR
         binary_op ::= BINARY_XOR
         binary_op ::= BINARY_SUBTRACT
-        binary_op ::= BINARY_DIVIDE
         binary_op ::= BINARY_TRUE_DIVIDE
         binary_op ::= BINARY_FLOOR_DIVIDE
         binary_op ::= BINARY_MODULO
@@ -442,21 +431,11 @@ class PythonParser(GenericASTBuilder):
         unary_op ::= UNARY_INVERT
 
         unary_not ::= expr UNARY_NOT
-        unary_convert ::= expr UNARY_CONVERT
 
         binary_subscr ::= expr expr BINARY_SUBSCR
-        binary_subscr2 ::= expr expr DUP_TOPX_2 BINARY_SUBSCR
 
         load_attr ::= expr LOAD_ATTR
         get_iter ::= expr GET_ITER
-        slice0 ::= expr SLICE+0
-        slice0 ::= expr DUP_TOP SLICE+0
-        slice1 ::= expr expr SLICE+1
-        slice1 ::= expr expr DUP_TOPX_2 SLICE+1
-        slice2 ::= expr expr SLICE+2
-        slice2 ::= expr expr DUP_TOPX_2 SLICE+2
-        slice3 ::= expr expr expr SLICE+3
-        slice3 ::= expr expr expr DUP_TOPX_3 SLICE+3
         buildslice3 ::= expr expr expr BUILD_SLICE_3
         buildslice2 ::= expr expr BUILD_SLICE_2
 
@@ -465,12 +444,6 @@ class PythonParser(GenericASTBuilder):
         _mklambda ::= load_closure mklambda
         _mklambda ::= mklambda
 
-        # Note: Python < 2.7 doesn't have *POP* or this. Remove from here?
-        # FIXME: segregate 2.7+
-
-        or   ::= expr JUMP_IF_TRUE_OR_POP expr COME_FROM
-        and  ::= expr JUMP_IF_FALSE_OR_POP expr COME_FROM
-
         or   ::= expr jmp_true expr come_from_opt
         and  ::= expr jmp_false expr come_from_opt
         and2 ::= _jump jmp_false COME_FROM expr COME_FROM
@@ -489,14 +462,6 @@ class PythonParser(GenericASTBuilder):
         ret_expr_or_cond ::= ret_cond
         ret_expr_or_cond ::= ret_cond_not
 
-        # Note: Python < 2.7 doesn't have *POP* or this. Remove from here?
-        # FIXME: segregate 2.7+
-
-        ret_and  ::= expr JUMP_IF_FALSE_OR_POP ret_expr_or_cond COME_FROM
-        ret_or   ::= expr JUMP_IF_TRUE_OR_POP ret_expr_or_cond COME_FROM
-        ret_cond ::= expr POP_JUMP_IF_FALSE expr RETURN_END_IF ret_expr_or_cond
-        ret_cond_not ::= expr POP_JUMP_IF_TRUE expr RETURN_END_IF ret_expr_or_cond
-
         stmt ::= return_lambda
         stmt ::= conditional_lambda
 
@@ -508,15 +473,9 @@ class PythonParser(GenericASTBuilder):
         compare ::= expr expr COMPARE_OP
         cmp_list ::= expr cmp_list1 ROT_TWO POP_TOP
                 _come_from
-        cmp_list1 ::= expr DUP_TOP ROT_THREE
-                COMPARE_OP JUMP_IF_FALSE_OR_POP
-                cmp_list1 COME_FROM
         cmp_list1 ::= expr DUP_TOP ROT_THREE
                 COMPARE_OP jmp_false
                 cmp_list1 _come_from
-        cmp_list1 ::= expr DUP_TOP ROT_THREE
-                COMPARE_OP JUMP_IF_FALSE_OR_POP
-                cmp_list2 COME_FROM
         cmp_list1 ::= expr DUP_TOP ROT_THREE
                 COMPARE_OP jmp_false
                 cmp_list2 _come_from
@@ -561,10 +520,6 @@ class PythonParser(GenericASTBuilder):
         designator ::= STORE_GLOBAL
         designator ::= STORE_DEREF
         designator ::= expr STORE_ATTR
-        designator ::= expr STORE_SLICE+0
-        designator ::= expr expr STORE_SLICE+1
-        designator ::= expr expr STORE_SLICE+2
-        designator ::= expr expr expr STORE_SLICE+3
         designator ::= store_subscr
         store_subscr ::= expr expr STORE_SUBSCR
         designator ::= unpack

uncompyle6/parsers/parse2.py

@@ -43,6 +43,9 @@ class Python2Parser(PythonParser):
     def p_stmt2(self, args):
         """
         while1stmt ::= SETUP_LOOP l_stmts_opt JUMP_BACK POP_BLOCK COME_FROM
+        exec_stmt ::= expr exprlist DUP_TOP EXEC_STMT
+        exec_stmt ::= expr exprlist EXEC_STMT
+
         """
 
     def p_print_to(self, args):
@@ -89,8 +92,6 @@ class Python2Parser(PythonParser):
         raise_stmt3 ::= expr expr expr RAISE_VARARGS_3
 
         stmt ::= exec_stmt
-        exec_stmt ::= expr exprlist DUP_TOP EXEC_STMT
-        exec_stmt ::= expr exprlist EXEC_STMT
 
         stmt ::= assert
         stmt ::= assert2
@@ -218,8 +219,13 @@ class Python2Parser(PythonParser):
 
 
     def p_expr2(self, args):
-        '''
+        """
         expr ::= LOAD_LOCALS
+        expr ::= slice0
+        expr ::= slice1
+        expr ::= slice2
+        expr ::= slice3
+        expr ::= unary_convert
 
         slice0 ::= expr SLICE+0
         slice0 ::= expr DUP_TOP SLICE+0
@@ -229,10 +235,38 @@ class Python2Parser(PythonParser):
         slice2 ::= expr expr DUP_TOPX_2 SLICE+2
         slice3 ::= expr expr expr SLICE+3
         slice3 ::= expr expr expr DUP_TOPX_3 SLICE+3
+        unary_convert ::= expr UNARY_CONVERT
 
         # In Python 3, DUP_TOPX_2 is DUP_TOP_TWO
         binary_subscr2 ::= expr expr DUP_TOPX_2 BINARY_SUBSCR
-        '''
+        """
+
+    def p_slice2(self, args):
+        """
+        designator ::= expr STORE_SLICE+0
+        designator ::= expr expr STORE_SLICE+1
+        designator ::= expr expr STORE_SLICE+2
+        designator ::= expr expr expr STORE_SLICE+3
+        augassign1 ::= expr expr inplace_op ROT_TWO   STORE_SLICE+0
+        augassign1 ::= expr expr inplace_op ROT_THREE STORE_SLICE+1
+        augassign1 ::= expr expr inplace_op ROT_THREE STORE_SLICE+2
+        augassign1 ::= expr expr inplace_op ROT_FOUR  STORE_SLICE+3
+        slice0 ::= expr SLICE+0
+        slice0 ::= expr DUP_TOP SLICE+0
+        slice1 ::= expr expr SLICE+1
+        slice1 ::= expr expr DUP_TOPX_2 SLICE+1
+        slice2 ::= expr expr SLICE+2
+        slice2 ::= expr expr DUP_TOPX_2 SLICE+2
+        slice3 ::= expr expr expr SLICE+3
+        slice3 ::= expr expr expr DUP_TOPX_3 SLICE+3
+        """
+
+    def p_op2(self, args):
+        """
+        inplace_op ::= INPLACE_DIVIDE
+        binary_op  ::= BINARY_DIVIDE
+        binary_subscr2 ::= expr expr DUP_TOPX_2 BINARY_SUBSCR
+        """
 
     def add_custom_rules(self, tokens, customize):
         '''

uncompyle6/parsers/parse3.py

@@ -90,10 +90,6 @@ class Python3Parser(PythonParser):
         raise_stmt2 ::= expr expr RAISE_VARARGS_2
         raise_stmt3 ::= expr expr expr RAISE_VARARGS_3
 
-        stmt ::= exec_stmt
-        exec_stmt ::= expr exprlist DUP_TOP EXEC_STMT
-        exec_stmt ::= expr exprlist EXEC_STMT
-
         stmt ::= assert
         stmt ::= assert2
         stmt ::= ifstmt
@@ -115,10 +111,6 @@ class Python3Parser(PythonParser):
         del_stmt ::= DELETE_FAST
         del_stmt ::= DELETE_NAME
         del_stmt ::= DELETE_GLOBAL
-        del_stmt ::= expr DELETE_SLICE+0
-        del_stmt ::= expr expr DELETE_SLICE+1
-        del_stmt ::= expr expr DELETE_SLICE+2
-        del_stmt ::= expr expr expr DELETE_SLICE+3
         del_stmt ::= delete_subscr
         delete_subscr ::= expr expr DELETE_SUBSCR
         del_stmt ::= expr DELETE_ATTR

uncompyle6/parsers/parse34.py

@@ -57,7 +57,7 @@ if __name__ == '__main__':
         s = get_scanner(PYTHON_VERSION, IS_PYPY)
         opcode_set = set(s.opc.opname).union(set(
             """JUMP_BACK CONTINUE RETURN_END_IF COME_FROM
-               LOAD_GENEXPR LOAD_ASSERT LOAD_SETCOMP LOAD_DICTCOMP
+               LOAD_GENEXPR LOAD_ASSERT LOAD_SETCOMP LOAD_DICTCOMP LOAD_CLASSNAME
                LAMBDA_MARKER RETURN_LAST
             """.split()))
         remain_tokens = set(tokens) - opcode_set