grammar isolation and reduction

pytest/test_grammar.py

@@ -20,8 +20,8 @@ def test_grammar():
     unused_rhs = set(['list', 'mkfunc',
                       'mklambda',
                       'unpack',])
-    expect_right_recursive = frozenset([('designList',
-                                         ('store', 'DUP_TOP', 'designList'))])
+    expect_right_recursive = set([('designList',
+                                   ('store', 'DUP_TOP', 'designList'))])
     if PYTHON3:
         expect_lhs.add('load_genexpr')
         expect_lhs.add('kvlist')
@@ -35,14 +35,21 @@ def test_grammar():
             expect_lhs.add("annotate_arg")
             expect_lhs.add("annotate_tuple")
             unused_rhs.add("mkfunc_annotate")
-            if PYTHON_VERSION != 3.6:
+            if PYTHON_VERSION < 3.6:
                 # 3.6 has at least one non-custom call rule
                 # the others don't
                 unused_rhs.add('call')
+                if PYTHON_VERSION == 3.5:
+                    expect_right_recursive.add((('l_stmts',
+                                                 ('lastl_stmt', 'COME_FROM', 'l_stmts'))))
+                    pass
+                pass
             else:
-                # These are custom rules set in 3.5
+                expect_right_recursive.add((('l_stmts',
+                                             ('lastl_stmt', 'COME_FROM', 'l_stmts'))))
                 unused_rhs.add('build_map_unpack_with_call')
                 unused_rhs.add('unmapexpr')
+                # expect_lhs.add('kwargs1')
                 pass
             pass
         pass

uncompyle6/parser.py

@@ -42,6 +42,7 @@ class PythonParser(GenericASTBuilder):
             'kvlist_n',
             # Python 3.6+
             'joined_str',
+            'come_from_loops',
             ]
         self.collect = frozenset(nt_list)
 
@@ -157,9 +158,6 @@ class PythonParser(GenericASTBuilder):
         else:
             raise ParserError(None, -1)
 
-    def typestring(self, token):
-        return token.kind
-
     def nonterminal(self, nt, args):
         n = len(args)
 
@@ -177,6 +175,7 @@ class PythonParser(GenericASTBuilder):
             rv.append(args[1])
         elif n == 1 and args[0] in self.singleton:
             rv = GenericASTBuilder.nonterminal(self, nt, args[0])
+            del args[0] # save memory
         else:
             rv = GenericASTBuilder.nonterminal(self, nt, args)
         return rv
@@ -225,7 +224,6 @@ class PythonParser(GenericASTBuilder):
         lastc_stmt ::= iflaststmt
         lastc_stmt ::= forelselaststmt
         lastc_stmt ::= ifelsestmtc
-        lastc_stmt ::= tryelsestmtc
 
         c_stmts_opt ::= c_stmts
         c_stmts_opt ::= passstmt

uncompyle6/parsers/parse2.py

@@ -133,6 +133,8 @@ class Python2Parser(PythonParser):
                             POP_BLOCK LOAD_CONST
                             COME_FROM suite_stmts_opt END_FINALLY
 
+        lastc_stmt ::= tryelsestmtc
+
         # Move to 2.7? 2.6 may use come_froms
         tryelsestmtc    ::= SETUP_EXCEPT suite_stmts_opt POP_BLOCK
                             except_handler else_suitec COME_FROM

uncompyle6/parsers/parse3.py

@@ -104,6 +104,7 @@ class Python3Parser(PythonParser):
 
         kwarg   ::= LOAD_CONST expr
         kwargs  ::= kwarg*
+        kwargs1 ::= kwarg+
 
         classdef ::= build_class store
 
@@ -268,7 +269,7 @@ class Python3Parser(PythonParser):
         except_handler ::= JUMP_FORWARD COME_FROM_EXCEPT except_stmts
                             END_FINALLY COME_FROM_EXCEPT_CLAUSE
 
-        for_block ::= l_stmts_opt come_from_loops JUMP_BACK
+        for_block ::= l_stmts_opt COME_FROM_LOOP JUMP_BACK
         for_block ::= l_stmts
         iflaststmtl ::= testexpr c_stmts_opt
         """
@@ -300,7 +301,6 @@ class Python3Parser(PythonParser):
         opt_come_from_except ::= come_from_except_clauses
 
         come_from_except_clauses ::= COME_FROM_EXCEPT_CLAUSE+
-        come_from_loops          ::= COME_FROM_LOOP*
         """
 
     def p_jump3(self, args):
@@ -335,7 +335,7 @@ class Python3Parser(PythonParser):
     def p_loop_stmt3(self, args):
         """
         forstmt           ::= SETUP_LOOP expr _for store for_block POP_BLOCK
-                              come_from_loops
+                              COME_FROM_LOOP
 
         forelsestmt       ::= SETUP_LOOP expr _for store for_block POP_BLOCK else_suite
                               COME_FROM_LOOP
@@ -812,7 +812,7 @@ class Python3Parser(PythonParser):
                                                  opname, token.attr, customize)
 
                 if args_kw > 0:
-                    kwargs_str = 'kwargs '
+                    kwargs_str = 'kwargs1 '
                 else:
                     kwargs_str = ''
 
@@ -907,6 +907,11 @@ class Python3Parser(PythonParser):
                     rule = ('mkfunc ::= kwargs %s%s %s' %
                             ('pos_arg ' * args_pos, 'LOAD_CONST '*2,
                              opname))
+                elif self.version > 3.5:
+                    # positional args before keyword args
+                    rule = ('mkfunc ::= %skwargs1 %s %s' %
+                            ('pos_arg ' * args_pos, 'LOAD_CONST '*2,
+                             opname))
                 elif self.version > 3.3:
                     # positional args before keyword args
                     rule = ('mkfunc ::= %skwargs %s %s' %

uncompyle6/parsers/parse35.py

@@ -85,7 +85,7 @@ class Python35Parser(Python34Parser):
                                POP_TOP POP_TOP POP_TOP POP_EXCEPT POP_BLOCK
                                JUMP_ABSOLUTE END_FINALLY COME_FROM
                                for_block POP_BLOCK JUMP_ABSOLUTE
-                               come_from_loops
+                               COME_FROM_LOOP
 
         async_for_stmt     ::= SETUP_LOOP expr
                                GET_AITER
@@ -97,7 +97,7 @@ class Python35Parser(Python34Parser):
                                POP_TOP POP_TOP POP_TOP POP_EXCEPT POP_BLOCK
                                JUMP_ABSOLUTE END_FINALLY JUMP_BACK
                                passstmt POP_BLOCK JUMP_ABSOLUTE
-                               come_from_loops
+                               COME_FROM_LOOP
 
         stmt               ::= async_forelse_stmt
         async_forelse_stmt ::= SETUP_LOOP expr

uncompyle6/parsers/parse36.py

@@ -37,6 +37,10 @@ class Python36Parser(Python35Parser):
         call          ::= expr expr CALL_FUNCTION_EX
         call          ::= expr expr expr CALL_FUNCTION_EX_KW_1
 
+        for_block       ::= l_stmts_opt come_from_loops JUMP_BACK
+        come_from_loops ::= COME_FROM_LOOP*
+
+
         # This might be valid in < 3.6
         and  ::= expr jmp_false expr